def testChainedDepends(self): '''Test chain dependent''' shutil.rmtree('.sos') os.makedirs('.sos/.runtime') script = SoS_Script(r''' # this step provides variable `var` [index: provides='{filename}.bam.bai'] input: "${filename}.bam" sh: echo "Generating ${output}" touch ${output} [call: provides='{filename}.vcf'] input: "${filename}.bam" depends: "${input}.bai" sh: echo "Calling variants from ${input} with ${depends} to ${output}" touch ${output} ''') FileTarget('a.bam.bai').remove('both') FileTarget('a.vcf').remove('both') self.touch('a.bam') Base_Executor(script.workflow()).run(targets=['a.vcf']) for file in ('a.vcf', 'a.bam', 'a.bam.bai'): FileTarget(file).remove('both')
def testSharedVar(self): '''Test shared var with rq queue''' script = SoS_Script(''' [work_1: shared = {'data': 'output'}] input: "1.txt", "2.txt", group_by = 'single', pattern = '{name}.{ext}' output: expand_pattern('{_name}.out') task: concurrent = True run: touch ${_output} [work_2] input: "1.txt", "2.txt", group_by = 'single', pattern = '{name}.{ext}', paired_with = ['data'] output: expand_pattern('{_name}.out2') task: concurrent = True run: touch ${_data} ${_output} [default] sos_run("work:1+work:2") ''') self.touch(['1.txt', '2.txt']) subprocess.call('sos remove . -t -y', shell=True) wf = script.workflow() RQ_Executor(wf).run() for f in ['1.out', '1.out2', '2.out', '2.out2']: self.assertTrue(FileTarget(f).exists('target')) FileTarget(f).remove('both')
def testPatternReuse(self): '''Test repeated use of steps that use pattern and produce different files.''' # for f in ['A1.txt', 'A2.txt', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p']: FileTarget(f).remove('both') # # A1 <- P <- B1 # A1 <- P <- B2 # A2 # script = SoS_Script(''' [A_1] input: 'B1.txt.p', 'B2.txt.p' output: 'A1.txt' sh: touch A1.txt [A_2] sh: touch A2.txt [B1: provides='B1.txt'] sh: touch B1.txt [B2: provides='B2.txt'] sh: touch B2.txt [P: provides='{filename}.p'] input: filename sh: touch ${output} ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow() dag = Base_Executor(wf).initialize_dag() self.assertDAG(dag, ''' strict digraph "" { "P ['B2.txt.p']"; "B1 ['B1.txt']"; "B2 ['B2.txt']"; A_2; A_1; "P ['B1.txt.p']"; "P ['B2.txt.p']" -> A_1; "B1 ['B1.txt']" -> "P ['B1.txt.p']"; "B2 ['B2.txt']" -> "P ['B2.txt.p']"; A_1 -> A_2; "P ['B1.txt.p']" -> A_1; } ''') Base_Executor(wf).run() for f in ['A1.txt', 'A2.txt', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p']: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both')
def testLiteralConnection(self): '''Testing the connection of steps with by variables.''' for f in ['A1.txt']: FileTarget(f).remove('both') # # A1 introduces a shared variable ss, A3 depends on ss but not A2 # script = SoS_Script(''' [A_1: shared='p'] sh: touch 'A1.txt' p = 'A1.txt' [A_2] input: None sh: sleep 3 [A_3] input: p sh: sleep 3 [A_4] input: p sh: sleep 3 [A_5] input: dynamic(p) ''') wf = script.workflow('A') dag = Base_Executor(wf).initialize_dag() self.assertDAG( dag, ''' strict digraph "" { A_1; A_4; A_2; A_3; A_5; A_1 -> A_4; A_1 -> A_3; A_1 -> A_5; A_4 -> A_5; } ''') env.max_jobs = 3 st = time.time() MP_Executor(wf).run() self.assertLess(time.time() - st, 5) for f in ['A1.txt']: self.assertTrue(FileTarget(f).exists()) FileTarget(f).remove('both')
def testParallelExecution(self): '''Test basic parallel execution''' ''' A1 <- None A2 <- B2 ''' for f in ['A1.txt', 'B2.txt', 'A2.txt']: FileTarget(f).remove('both') script = SoS_Script(''' [A_1] output: 'A1.txt' sh: sleep 3 touch A1.txt [A_2] input: 'B2.txt' output: 'A2.txt' sh: sleep 3 touch A2.txt [B: provides='B2.txt'] output: 'B2.txt' sh: touch B2.txt ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow() dag = Base_Executor(wf).initialize_dag() self.assertDAG( dag, ''' strict digraph "" { A_1; A_2; "B ['B2.txt']"; "B ['B2.txt']" -> A_2; } ''') env.max_jobs = 4 st = time.time() #env.verbosity = 4 MP_Executor(wf).run() self.assertLess(time.time() - st, 4) for f in ['A1.txt', 'B2.txt', 'A2.txt']: FileTarget(f).remove('both')
def testDynamicNestedWorkflow(self): # # Because we are not sure which workflows would be executed # until run time, the DAG should not contain nested workflow # until runtime. # for f in [ 'B0.txt', 'B0.txt.p', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p' ]: FileTarget(f).remove('both') # # A1 <- P <- B # A1 <- P <- B # A2 # # ALL calls A and B with parameter # script = SoS_Script(''' [A_1] parameter: num = 2 input: "B${num}.txt.p" [B: provides='B{num}.txt'] sh: touch 'B${num}.txt' [P: provides='{filename}.p'] input: filename sh: touch ${output} [ALL] for num in range(3): sos_run('A') ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow('ALL') Base_Executor(wf).run() for f in [ 'B0.txt', 'B0.txt.p', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p' ]: self.assertTrue(FileTarget(f).exists()) FileTarget(f).remove('both')
def testOverwriteKeyword(self): '''Test overwrite sos keyword with user defined one.''' FileTarget('a.txt').remove('both') # script = SoS_Script(''' def run(script): pass [1] run: touch a.txt ''') wf = script.workflow() Base_Executor(wf).run() self.assertFalse(os.path.isfile('a.txt')) # script = SoS_Script(''' parameter: run = 5 [1] run: touch a.txt ''') wf = script.workflow() self.assertRaises(Exception, Base_Executor(wf).run)
def testSharedDependency(self): # # shared variable should introduce additional dependency # for f in ['A1.txt']: FileTarget(f).remove('both') # # A1 introduces a shared variable ss, A3 depends on ss but not A2 # script = SoS_Script(''' [A_1: shared='ss'] ss = 'A1' [A_2] input: None sh: sleep 3 [A_3] input: None import time time.sleep(3) with open("${ss}.txt", 'w') as tmp: tmp.write('test') ''') wf = script.workflow('A') dag = Base_Executor(wf).initialize_dag() self.assertDAG( dag, ''' strict digraph "" { A_3; A_1; A_2; A_1 -> A_3; } ''') env.max_jobs = 3 st = time.time() MP_Executor(wf).run() self.assertLess(time.time() - st, 5) for f in ['A1.txt']: self.assertTrue(FileTarget(f).exists()) FileTarget(f).remove('both')
def testSoSRun(self): '''Test action sos_run with keyword parameters''' for f in ['0.txt', '1.txt']: FileTarget(f).remove('both') script = SoS_Script(r''' [A] parameter: num=5 sh: touch ${num}.txt [batch] for k in range(2): sos_run('A', num=k) ''') env.verbosity = 3 wf = script.workflow('batch') Base_Executor(wf).run() for f in ['0.txt', '1.txt']: self.assertTrue(FileTarget(f).exists()) FileTarget(f).remove('both')
def testArgs(self): '''Test args option of scripts''' FileTarget('a.txt').remove('both') script = SoS_Script(r''' [0] sh: args='-n' touch a.txt ''') wf = script.workflow() Base_Executor(wf).run() self.assertFalse(os.path.exists('a.txt'))
def testReverseSharedVariable(self): '''Test shared variables defined in auxiliary steps''' FileTarget('a.txt').remove('both') script = SoS_Script(r''' [A: shared='b', provides='a.txt'] b = 1 sh: touch a.txt [B_1] depends: 'a.txt' [B_2] print(b) ''') wf = script.workflow('B') Base_Executor(wf).run() self.assertTrue(env.sos_dict['b'], 1)
def testPandoc(self): '''Test action pandoc''' if not shutil.which('pandoc'): return script = SoS_Script(r''' [10] report: ## Some random figure Generated by matplotlib [100] # generate report output: 'myreport.html' pandoc(output=_output[0], to='html') ''') wf = script.workflow() Base_Executor(wf).run() self.assertTrue(os.path.isfile('myreport.html')) # FileTarget('myreport.html').remove('both')
def testIncludedNestedWorkFlow(self): '''Test the source option of sos_run''' # nested subworkflow with step option and others self.touch(['a.txt', 'b.txt']) # shutil.rmtree('.sos') os.makedirs('.sos/.runtime') with open('inc.sos', 'w') as sos: sos.write(''' # test sos script # global definition GLB = 5 parameter: parB = 10 [A_1: shared='executed'] executed.append('t.' + step_name) output: _input[0] + '.a1' sh: touch ${output} [A_2: shared='executed'] executed.append('t.' + step_name) output: _input[0] + '.a2' sh: touch ${output} ''') script = SoS_Script(''' %from inc include * if 'executed' not in locals(): executed = [] [b_1: skip=False, shared='executed'] executed.append(step_name) input: 'a.txt', 'b.txt', group_by='single' sos_run('A') ''') wf = script.workflow('b') Base_Executor(wf).run() self.assertEqual(env.sos_dict['GLB'], 5) self.assertEqual(env.sos_dict['parB'], 10) self.assertEqual(env.sos_dict['executed'], ['b_1', 't.A_1', 't.A_2', 't.A_1', 't.A_2']) # shutil.rmtree('.sos') os.makedirs('.sos/.runtime') for file in ('a.txt.a1', 'a.txt.a1.a2', 'b.txt.a1', 'b.txt.a1.a2'): FileTarget(file).remove('both') script = SoS_Script(''' %include inc as k if 'executed' not in locals(): executed = [] [b_1: skip=False, shared='executed'] executed.append('g.' + step_name) input: 'a.txt', 'b.txt', group_by='single' sos_run('k.A') ''') wf = script.workflow('b') Base_Executor(wf).run() self.assertEqual(env.sos_dict['k'].GLB, 5) self.assertEqual(env.sos_dict['k'].parB, 10) self.assertEqual(env.sos_dict['executed'], ['g.b_1', 't.k.A_1', 't.k.A_2', 't.k.A_1', 't.k.A_2']) # os.remove('inc.sos')
def tearDown(self): # for f in self.temp_files: FileTarget(f).remove('both') subprocess.call('redis-cli shutdown', shell=True)
def testNestedWorkflow(self): '''Test the creation and execution of combined workfow''' self.touch(['a.txt', 'b.txt', 'b.begin']) script = SoS_Script(''' if 'executed' not in locals(): executed = [] if 'inputs' not in locals(): inputs = [] [a_1: shared=['executed', 'inputs']] executed.append(step_name) inputs.append(input) [a_2: shared=['executed', 'inputs']] executed.append(step_name) inputs.append(input) [a_3: shared=['executed', 'inputs']] executed.append(step_name) inputs.append(input) [a_4: shared=['executed', 'inputs']] executed.append(step_name) output: 'a.done' inputs.append(input) sh: touch ${output} [b_1: shared=['executed', 'inputs']] executed.append(step_name) input: 'b.begin' inputs.append(input) [b_2: shared=['executed', 'inputs']] executed.append(step_name) inputs.append(input) [b_3: shared=['executed', 'inputs']] executed.append(step_name) inputs.append(input) [b_4: shared=['executed', 'inputs']] executed.append(step_name) output: 'b.txt' inputs.append(input) [c: shared=['executed', 'inputs']] executed.append(step_name) input: 'a.txt' output: 'b.txt' inputs.append(input) sos_run('a+b') ''') env.sig_mode = 'ignore' wf = script.workflow('c') Base_Executor(wf).run() # order of execution is not guaranteed self.assertEqual(sorted(env.sos_dict['executed']), sorted(['c_0', 'a_1', 'a_2', 'a_3', 'a_4', 'b_1', 'b_2', 'b_3', 'b_4'])) # step will be looped self.touch(['a.txt', 'b.txt']) script = SoS_Script(''' if 'executed' not in locals(): executed = [] if 'inputs' not in locals(): inputs = [] [a_1:shared=['executed', 'inputs']] executed.append(step_name) output: _input[0] + '.a1' inputs.append(input) sh: touch ${output} [a_2:shared=['executed', 'inputs']] executed.append(step_name) output: _input[0] + '.a2' inputs.append(input) sh: touch ${output} [c:shared=['executed', 'inputs']] executed.append(step_name) input: 'a.txt', 'b.txt', group_by='single' inputs.append(_input) sos_run('a') ''') wf = script.workflow('c') Base_Executor(wf).run() self.assertEqual(env.sos_dict['executed'], ['c_0', 'a_1', 'a_2', 'a_1', 'a_2']) #self.assertEqual(env.sos_dict['inputs'], [['a.txt'], ['a.txt'], ['a.txt.a1'], ['b.txt'], ['b.txt'], ['b.txt.a1']]) for file in ('a.txt.a1', 'a.txt.a1.a2', 'b.txt.a1', 'b.txt.a1.a2'): FileTarget(file).remove('both') # # allow specifying a single step # step will be looped script = SoS_Script(''' if 'executed' not in locals(): executed = [] [a_1:shared='executed'] executed.append(step_name) [a_2:shared='executed'] executed.append(step_name) [c_0:shared='executed'] executed.append(step_name) [c_1:shared='executed'] executed.append(step_name) input: 'a.txt', 'b.txt', group_by='single' sos_run('a:2') ''') wf = script.workflow('c') Base_Executor(wf).run() self.assertEqual(env.sos_dict['executed'], ['c_0', 'c_1', 'a_2', 'a_2']) # allow specifying a single step # step will be looped script = SoS_Script(''' if 'executed' not in locals(): executed = [] [a_1:shared='executed'] executed.append(step_name) [a_2:shared='executed'] executed.append(step_name) [c_0:shared='executed'] executed.append(step_name) [c_1:shared='executed'] executed.append(step_name) input: 'a.txt', 'b.txt', group_by='single' sos_run('a:2') ''') wf = script.workflow('c') Base_Executor(wf).run() self.assertEqual(env.sos_dict['executed'], ['c_0', 'c_1', 'a_2', 'a_2']) # # recursive subworkflow not allowed script = SoS_Script(''' if 'executed' not in locals(): executed = [] [a_1:shared='executed'] executed.append(step_name) [a_2:shared='executed'] executed.append(step_name) [c_0:shared='executed'] executed.append(step_name) [c_1:shared='executed'] executed.append(step_name) input: 'a.txt', 'b.txt', group_by='single' sos_run('a_2+c') ''') wf = script.workflow('c') self.assertRaises(ExecuteError, Base_Executor(wf).run) # # nested subworkflow is allowed script = SoS_Script(''' if 'executed' not in locals(): executed = [] [a_1:shared='executed'] executed.append(step_name) [a_2:shared='executed'] executed.append(step_name) [a_3:shared='executed'] executed.append(step_name) [b_1:shared='executed'] executed.append(step_name) [b_2:shared='executed'] executed.append(step_name) sos_run('a:1-2') [c_0:shared='executed'] executed.append(step_name) [c_1:shared='executed'] executed.append(step_name) input: 'a.txt' sos_run('a+b') ''') wf = script.workflow('c') Base_Executor(wf).run() self.assertEqual(env.sos_dict['executed'], ['c_0', 'c_1', 'a_1', 'a_2', 'a_3', 'b_1', 'b_2', 'a_1', 'a_2']) # # # nested subworkflow with step option and others script = SoS_Script(''' if 'executed' not in locals(): executed = [] [a_1:shared='executed'] executed.append(step_name) [a_2:shared='executed'] executed.append(step_name) [a_3:shared='executed'] executed.append(step_name) [b:shared='executed'] executed.append(step_name) input: 'a.txt', 'b.txt', group_by='single' sos_run('a:3+a:1') [d:shared='executed'] executed.append(step_name) input: 'a.txt', 'b.txt', group_by='single' sos_run('a:2') [e2_2:shared='executed'] executed.append(step_name) input: 'a.txt', 'b.txt', group_by='single' ''') wf = script.workflow('b') Base_Executor(wf).run() self.assertEqual(env.sos_dict['executed'], ['b_0', 'a_3', 'a_1', 'a_3', 'a_1']) wf = script.workflow('d') Base_Executor(wf).run() self.assertEqual(env.sos_dict['executed'], ['d_0', 'a_2', 'a_2']) wf = script.workflow('e2') Base_Executor(wf).run() self.assertEqual(env.sos_dict['executed'], ['e2_2']) # # clean up FileTarget('a.done').remove('both')
def testLongChain(self): '''Test long make file style dependencies.''' # for f in [ 'A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt' ]: FileTarget(f).remove('both') # # A1 <- B1 <- B2 <- B3 # | # | # \/ # A2 <- B2 <- C1 <- C2 <- C4 # C3 # script = SoS_Script(''' [A_1] input: 'B1.txt' output: 'A1.txt' sh: touch A1.txt [A_2] depends: 'B2.txt' sh: touch A2.txt [B1: provides='B1.txt'] depends: 'B2.txt' sh: touch B1.txt [B2: provides='B2.txt'] depends: 'B3.txt', 'C1.txt' sh: touch B2.txt [B3: provides='B3.txt'] sh: touch B3.txt [C1: provides='C1.txt'] depends: 'C2.txt', 'C3.txt' sh: touch C1.txt [C2: provides='C2.txt'] depends: 'C4.txt' sh: touch C2.txt [C3: provides='C3.txt'] depends: 'C4.txt' sh: touch C3.txt [C4: provides='C4.txt'] sh: touch C4.txt ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow() dag = Base_Executor(wf).initialize_dag() self.assertDAG( dag, ''' strict digraph "" { "C4 ['C4.txt']"; "B1 ['B1.txt']"; "C1 ['C1.txt']"; "C2 ['C2.txt']"; "C3 ['C3.txt']"; A_1; "B2 ['B2.txt']"; "B3 ['B3.txt']"; A_2; "C4 ['C4.txt']" -> "C2 ['C2.txt']"; "C4 ['C4.txt']" -> "C3 ['C3.txt']"; "B1 ['B1.txt']" -> A_1; "C1 ['C1.txt']" -> "B2 ['B2.txt']"; "C2 ['C2.txt']" -> "C1 ['C1.txt']"; "C3 ['C3.txt']" -> "C1 ['C1.txt']"; A_1 -> A_2; "B2 ['B2.txt']" -> "B1 ['B1.txt']"; "B2 ['B2.txt']" -> A_2; "B3 ['B3.txt']" -> "B2 ['B2.txt']"; } ''') Base_Executor(wf).run() for f in [ 'A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt' ]: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both')
def testTarget(self): '''Test executing only part of a workflow.''' # for f in ['A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']: FileTarget(f).remove('both') # # A1 <- B1 <- B2 <- B3 # | # | # \/ # A2 <- B2 <- C1 <- C2 <- C4 # C3 # script = SoS_Script(''' [A_1] input: 'B1.txt' output: 'A1.txt' sh: touch A1.txt [A_2] depends: 'B2.txt' sh: touch A2.txt [B1: provides='B1.txt'] depends: 'B2.txt' sh: touch B1.txt [B2: provides='B2.txt'] depends: 'B3.txt', 'C1.txt' sh: touch B2.txt [B3: provides='B3.txt'] sh: touch B3.txt [C1: provides='C1.txt'] depends: 'C2.txt', 'C3.txt' sh: touch C1.txt [C2: provides='C2.txt'] depends: 'C4.txt' sh: touch C2.txt [C3: provides='C3.txt'] depends: 'C4.txt' sh: touch C3.txt [C4: provides='C4.txt'] sh: touch C4.txt ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow() # # test 1, we only need to generate target 'B1.txt' dag = Base_Executor(wf).initialize_dag(targets=['B1.txt']) # note that A2 is no longer mentioned self.assertDAG(dag, ''' strict digraph "" { "B3 ['B3.txt']"; "C4 ['C4.txt']"; "C2 ['C2.txt']"; "C1 ['C1.txt']"; "B1 ['B1.txt']"; "B2 ['B2.txt']"; "C3 ['C3.txt']"; "B3 ['B3.txt']" -> "B2 ['B2.txt']"; "C4 ['C4.txt']" -> "C3 ['C3.txt']"; "C4 ['C4.txt']" -> "C2 ['C2.txt']"; "C2 ['C2.txt']" -> "C1 ['C1.txt']"; "C1 ['C1.txt']" -> "B2 ['B2.txt']"; "B2 ['B2.txt']" -> "B1 ['B1.txt']"; "C3 ['C3.txt']" -> "C1 ['C1.txt']"; } ''') Base_Executor(wf).run(targets=['B1.txt']) for f in ['A1.txt', 'A2.txt']: self.assertFalse(FileTarget(f).exists()) for f in ['C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both') # # test 2, we would like to generate two files dag = Base_Executor(wf).initialize_dag(targets=['B2.txt', 'C2.txt']) # note that A2 is no longer mentioned self.assertDAG(dag, ''' strict digraph "" { "C4 ['C4.txt']"; "B2 ['B2.txt']"; "C3 ['C3.txt']"; "B3 ['B3.txt']"; "C2 ['C2.txt']"; "C1 ['C1.txt']"; "C4 ['C4.txt']" -> "C2 ['C2.txt']"; "C4 ['C4.txt']" -> "C3 ['C3.txt']"; "C3 ['C3.txt']" -> "C1 ['C1.txt']"; "B3 ['B3.txt']" -> "B2 ['B2.txt']"; "C2 ['C2.txt']" -> "C1 ['C1.txt']"; "C1 ['C1.txt']" -> "B2 ['B2.txt']"; } ''') Base_Executor(wf).run(targets=['B2.txt', 'C2.txt']) for f in ['A1.txt', 'B1.txt', 'A2.txt']: self.assertFalse(FileTarget(f).exists()) for f in ['C2.txt', 'B2.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both') # # test 3, generate two separate trees # dag = Base_Executor(wf).initialize_dag(targets=['B3.txt', 'C2.txt']) # note that A2 is no longer mentioned self.assertDAG(dag, ''' strict digraph "" { "B3 ['B3.txt']"; "C2 ['C2.txt']"; "C4 ['C4.txt']"; "C4 ['C4.txt']" -> "C2 ['C2.txt']"; } ''') Base_Executor(wf).run(targets=['B3.txt', 'C2.txt']) for f in ['A1.txt', 'B1.txt', 'A2.txt', 'B2.txt', 'C1.txt', 'C3.txt']: self.assertFalse(FileTarget(f).exists()) for f in ['C2.txt', 'B3.txt', 'C4.txt']: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both')
def tearDown(self): for f in self.temp_files: FileTarget(f).remove('both')
def testTarget(self): '''Test executing only part of a workflow.''' # for f in [ 'A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt' ]: FileTarget(f).remove('both') # # A1 <- B1 <- B2 <- B3 # | # | # \/ # A2 <- B2 <- C1 <- C2 <- C4 # C3 # script = SoS_Script(''' [A_1] input: 'B1.txt' output: 'A1.txt' sh: touch A1.txt [A_2] depends: 'B2.txt' sh: touch A2.txt [B1: provides='B1.txt'] depends: 'B2.txt' sh: touch B1.txt [B2: provides='B2.txt'] depends: 'B3.txt', 'C1.txt' sh: touch B2.txt [B3: provides='B3.txt'] sh: touch B3.txt [C1: provides='C1.txt'] depends: 'C2.txt', 'C3.txt' sh: touch C1.txt [C2: provides='C2.txt'] depends: 'C4.txt' sh: touch C2.txt [C3: provides='C3.txt'] depends: 'C4.txt' sh: touch C3.txt [C4: provides='C4.txt'] sh: touch C4.txt ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow() # # test 1, we only need to generate target 'B1.txt' dag = Base_Executor(wf).initialize_dag(targets=['B1.txt']) # note that A2 is no longer mentioned self.assertDAG( dag, ''' strict digraph "" { "B3 ['B3.txt']"; "C4 ['C4.txt']"; "C2 ['C2.txt']"; "C1 ['C1.txt']"; "B1 ['B1.txt']"; "B2 ['B2.txt']"; "C3 ['C3.txt']"; "B3 ['B3.txt']" -> "B2 ['B2.txt']"; "C4 ['C4.txt']" -> "C3 ['C3.txt']"; "C4 ['C4.txt']" -> "C2 ['C2.txt']"; "C2 ['C2.txt']" -> "C1 ['C1.txt']"; "C1 ['C1.txt']" -> "B2 ['B2.txt']"; "B2 ['B2.txt']" -> "B1 ['B1.txt']"; "C3 ['C3.txt']" -> "C1 ['C1.txt']"; } ''') Base_Executor(wf).run(targets=['B1.txt']) for f in ['A1.txt', 'A2.txt']: self.assertFalse(FileTarget(f).exists()) for f in [ 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt' ]: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both') # # test 2, we would like to generate two files dag = Base_Executor(wf).initialize_dag(targets=['B2.txt', 'C2.txt']) # note that A2 is no longer mentioned self.assertDAG( dag, ''' strict digraph "" { "C4 ['C4.txt']"; "B2 ['B2.txt']"; "C3 ['C3.txt']"; "B3 ['B3.txt']"; "C2 ['C2.txt']"; "C1 ['C1.txt']"; "C4 ['C4.txt']" -> "C2 ['C2.txt']"; "C4 ['C4.txt']" -> "C3 ['C3.txt']"; "C3 ['C3.txt']" -> "C1 ['C1.txt']"; "B3 ['B3.txt']" -> "B2 ['B2.txt']"; "C2 ['C2.txt']" -> "C1 ['C1.txt']"; "C1 ['C1.txt']" -> "B2 ['B2.txt']"; } ''') Base_Executor(wf).run(targets=['B2.txt', 'C2.txt']) for f in ['A1.txt', 'B1.txt', 'A2.txt']: self.assertFalse(FileTarget(f).exists()) for f in ['C2.txt', 'B2.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both') # # test 3, generate two separate trees # dag = Base_Executor(wf).initialize_dag(targets=['B3.txt', 'C2.txt']) # note that A2 is no longer mentioned self.assertDAG( dag, ''' strict digraph "" { "B3 ['B3.txt']"; "C2 ['C2.txt']"; "C4 ['C4.txt']"; "C4 ['C4.txt']" -> "C2 ['C2.txt']"; } ''') Base_Executor(wf).run(targets=['B3.txt', 'C2.txt']) for f in ['A1.txt', 'B1.txt', 'A2.txt', 'B2.txt', 'C1.txt', 'C3.txt']: self.assertFalse(FileTarget(f).exists()) for f in ['C2.txt', 'B3.txt', 'C4.txt']: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both')
def testPatternReuse(self): '''Test repeated use of steps that use pattern and produce different files.''' # for f in [ 'A1.txt', 'A2.txt', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p' ]: FileTarget(f).remove('both') # # A1 <- P <- B1 # A1 <- P <- B2 # A2 # script = SoS_Script(''' [A_1] input: 'B1.txt.p', 'B2.txt.p' output: 'A1.txt' sh: touch A1.txt [A_2] sh: touch A2.txt [B1: provides='B1.txt'] sh: touch B1.txt [B2: provides='B2.txt'] sh: touch B2.txt [P: provides='{filename}.p'] input: filename sh: touch ${output} ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow() dag = Base_Executor(wf).initialize_dag() self.assertDAG( dag, ''' strict digraph "" { "P ['B2.txt.p']"; "B1 ['B1.txt']"; "B2 ['B2.txt']"; A_2; A_1; "P ['B1.txt.p']"; "P ['B2.txt.p']" -> A_1; "B1 ['B1.txt']" -> "P ['B1.txt.p']"; "B2 ['B2.txt']" -> "P ['B2.txt.p']"; A_1 -> A_2; "P ['B1.txt.p']" -> A_1; } ''') Base_Executor(wf).run() for f in [ 'A1.txt', 'A2.txt', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p' ]: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both')
def tearDown(self): for f in self.temp_files: FileTarget(f).remove('both') os.chdir(self.olddir)
def testLongChain(self): '''Test long make file style dependencies.''' # for f in ['A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']: FileTarget(f).remove('both') # # A1 <- B1 <- B2 <- B3 # | # | # \/ # A2 <- B2 <- C1 <- C2 <- C4 # C3 # script = SoS_Script(''' [A_1] input: 'B1.txt' output: 'A1.txt' sh: touch A1.txt [A_2] depends: 'B2.txt' sh: touch A2.txt [B1: provides='B1.txt'] depends: 'B2.txt' sh: touch B1.txt [B2: provides='B2.txt'] depends: 'B3.txt', 'C1.txt' sh: touch B2.txt [B3: provides='B3.txt'] sh: touch B3.txt [C1: provides='C1.txt'] depends: 'C2.txt', 'C3.txt' sh: touch C1.txt [C2: provides='C2.txt'] depends: 'C4.txt' sh: touch C2.txt [C3: provides='C3.txt'] depends: 'C4.txt' sh: touch C3.txt [C4: provides='C4.txt'] sh: touch C4.txt ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow() dag = Base_Executor(wf).initialize_dag() self.assertDAG(dag, ''' strict digraph "" { "C4 ['C4.txt']"; "B1 ['B1.txt']"; "C1 ['C1.txt']"; "C2 ['C2.txt']"; "C3 ['C3.txt']"; A_1; "B2 ['B2.txt']"; "B3 ['B3.txt']"; A_2; "C4 ['C4.txt']" -> "C2 ['C2.txt']"; "C4 ['C4.txt']" -> "C3 ['C3.txt']"; "B1 ['B1.txt']" -> A_1; "C1 ['C1.txt']" -> "B2 ['B2.txt']"; "C2 ['C2.txt']" -> "C1 ['C1.txt']"; "C3 ['C3.txt']" -> "C1 ['C1.txt']"; A_1 -> A_2; "B2 ['B2.txt']" -> "B1 ['B1.txt']"; "B2 ['B2.txt']" -> A_2; "B3 ['B3.txt']" -> "B2 ['B2.txt']"; } ''') Base_Executor(wf).run() for f in ['A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both')