Example #1
0
File: test_dag.py Project: mr-c/SOS
    def testChainedDepends(self):
        '''Test chain dependent'''
        shutil.rmtree('.sos')
        os.makedirs('.sos/.runtime')
        script = SoS_Script(r'''
# this step provides variable `var`
[index: provides='{filename}.bam.bai']
input: "${filename}.bam"
sh:
   echo "Generating ${output}"
   touch ${output}

[call: provides='{filename}.vcf']
input:   "${filename}.bam"
depends: "${input}.bai"
sh:
   echo "Calling variants from ${input} with ${depends} to ${output}"
   touch ${output}
''')
        FileTarget('a.bam.bai').remove('both')
        FileTarget('a.vcf').remove('both')
        self.touch('a.bam')
        Base_Executor(script.workflow()).run(targets=['a.vcf'])
        for file in ('a.vcf', 'a.bam', 'a.bam.bai'):
            FileTarget(file).remove('both')
Example #2
0
    def testSharedVar(self):
        '''Test shared var with rq queue'''
        script = SoS_Script('''
[work_1: shared = {'data': 'output'}]
input: "1.txt", "2.txt", group_by = 'single', pattern = '{name}.{ext}'
output: expand_pattern('{_name}.out')
task: concurrent = True
run:
  touch ${_output}

[work_2]
input: "1.txt", "2.txt", group_by = 'single', pattern = '{name}.{ext}', paired_with = ['data']
output: expand_pattern('{_name}.out2')
task: concurrent = True
run:
  touch ${_data} ${_output}

[default]
sos_run("work:1+work:2")
''')
        self.touch(['1.txt', '2.txt'])
        subprocess.call('sos remove . -t -y', shell=True)
        wf = script.workflow()
        RQ_Executor(wf).run()
        for f in ['1.out', '1.out2', '2.out', '2.out2']:
            self.assertTrue(FileTarget(f).exists('target'))
            FileTarget(f).remove('both')
Example #3
0
    def testPatternReuse(self):
        '''Test repeated use of steps that use pattern and produce different files.'''
        #
        for f in ['A1.txt', 'A2.txt', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p']:
            FileTarget(f).remove('both')
        #
        #  A1 <- P <- B1
        #  A1 <- P <- B2
        #  A2
        #
        script = SoS_Script('''
[A_1]
input: 'B1.txt.p', 'B2.txt.p'
output: 'A1.txt'
sh:
    touch A1.txt

[A_2]
sh:
    touch A2.txt

[B1: provides='B1.txt']
sh:
    touch B1.txt

[B2: provides='B2.txt']
sh:
    touch B2.txt

[P: provides='{filename}.p']
input: filename
sh:
    touch ${output}
''')
        # the workflow should call step K for step C_2, but not C_3
        wf = script.workflow()
        dag = Base_Executor(wf).initialize_dag()
        self.assertDAG(dag,
'''
strict digraph "" {
"P ['B2.txt.p']";
"B1 ['B1.txt']";
"B2 ['B2.txt']";
A_2;
A_1;
"P ['B1.txt.p']";
"P ['B2.txt.p']" -> A_1;
"B1 ['B1.txt']" -> "P ['B1.txt.p']";
"B2 ['B2.txt']" -> "P ['B2.txt.p']";
A_1 -> A_2;
"P ['B1.txt.p']" -> A_1;
}
''')
        Base_Executor(wf).run()
        for f in ['A1.txt', 'A2.txt', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p']:
            t = FileTarget(f)
            self.assertTrue(t.exists())
            t.remove('both')
Example #4
0
File: test_dag.py Project: mr-c/SOS
    def testLiteralConnection(self):
        '''Testing the connection of steps with by variables.'''
        for f in ['A1.txt']:
            FileTarget(f).remove('both')
        #
        # A1 introduces a shared variable ss, A3 depends on ss but not A2
        #
        script = SoS_Script('''
[A_1: shared='p']
sh:
    touch 'A1.txt'

p = 'A1.txt'

[A_2]
input: None

sh:
    sleep 3

[A_3]
input: p

sh:
    sleep 3

[A_4]
input: p
sh:
    sleep 3

[A_5]
input: dynamic(p)
''')
        wf = script.workflow('A')
        dag = Base_Executor(wf).initialize_dag()
        self.assertDAG(
            dag, '''
strict digraph "" {
A_1;
A_4;
A_2;
A_3;
A_5;
A_1 -> A_4;
A_1 -> A_3;
A_1 -> A_5;
A_4 -> A_5;
}
''')
        env.max_jobs = 3
        st = time.time()
        MP_Executor(wf).run()
        self.assertLess(time.time() - st, 5)
        for f in ['A1.txt']:
            self.assertTrue(FileTarget(f).exists())
            FileTarget(f).remove('both')
Example #5
0
File: test_dag.py Project: mr-c/SOS
    def testParallelExecution(self):
        '''Test basic parallel execution'''
        '''
        A1 <- None
        A2 <- B2
        '''
        for f in ['A1.txt', 'B2.txt', 'A2.txt']:
            FileTarget(f).remove('both')
        script = SoS_Script('''
[A_1]
output: 'A1.txt'
sh:
    sleep 3
    touch A1.txt

[A_2]
input:  'B2.txt'
output: 'A2.txt'
sh:
    sleep 3
    touch A2.txt

[B: provides='B2.txt']
output: 'B2.txt'
sh:
    touch B2.txt


''')
        # the workflow should call step K for step C_2, but not C_3
        wf = script.workflow()
        dag = Base_Executor(wf).initialize_dag()
        self.assertDAG(
            dag, '''
strict digraph "" {
A_1;
A_2;
"B ['B2.txt']";
"B ['B2.txt']" -> A_2;
}
''')
        env.max_jobs = 4
        st = time.time()
        #env.verbosity = 4
        MP_Executor(wf).run()
        self.assertLess(time.time() - st, 4)
        for f in ['A1.txt', 'B2.txt', 'A2.txt']:
            FileTarget(f).remove('both')
Example #6
0
File: test_dag.py Project: mr-c/SOS
    def testDynamicNestedWorkflow(self):
        #
        # Because we are not sure which workflows would be executed
        # until run time, the DAG should not contain nested workflow
        # until runtime.
        #
        for f in [
                'B0.txt', 'B0.txt.p', 'B1.txt', 'B1.txt.p', 'B2.txt',
                'B2.txt.p'
        ]:
            FileTarget(f).remove('both')
        #
        #  A1 <- P <- B
        #  A1 <- P <- B
        #  A2
        #
        #  ALL calls A and B with parameter
        #
        script = SoS_Script('''
[A_1]
parameter: num = 2
input: "B${num}.txt.p"

[B: provides='B{num}.txt']
sh:
    touch 'B${num}.txt'

[P: provides='{filename}.p']
input: filename
sh:
    touch ${output}

[ALL]

for num in range(3):
    sos_run('A')


''')
        # the workflow should call step K for step C_2, but not C_3
        wf = script.workflow('ALL')
        Base_Executor(wf).run()
        for f in [
                'B0.txt', 'B0.txt.p', 'B1.txt', 'B1.txt.p', 'B2.txt',
                'B2.txt.p'
        ]:
            self.assertTrue(FileTarget(f).exists())
            FileTarget(f).remove('both')
Example #7
0
    def testOverwriteKeyword(self):
        '''Test overwrite sos keyword with user defined one.'''
        FileTarget('a.txt').remove('both')
        #
        script = SoS_Script('''
def run(script):
    pass

[1]
run:
    touch a.txt
''')
        wf = script.workflow()
        Base_Executor(wf).run()
        self.assertFalse(os.path.isfile('a.txt'))
        #
        script = SoS_Script('''
parameter: run = 5

[1]
run:
    touch a.txt
''')
        wf = script.workflow()
        self.assertRaises(Exception, Base_Executor(wf).run)
Example #8
0
File: test_dag.py Project: mr-c/SOS
    def testSharedDependency(self):
        #
        # shared variable should introduce additional dependency
        #
        for f in ['A1.txt']:
            FileTarget(f).remove('both')
        #
        # A1 introduces a shared variable ss, A3 depends on ss but not A2
        #
        script = SoS_Script('''
[A_1: shared='ss']
ss = 'A1'

[A_2]
input: None

sh:
    sleep 3

[A_3]
input: None
import time
time.sleep(3)
with open("${ss}.txt", 'w') as tmp:
    tmp.write('test')

''')
        wf = script.workflow('A')
        dag = Base_Executor(wf).initialize_dag()
        self.assertDAG(
            dag, '''
strict digraph "" {
A_3;
A_1;
A_2;
A_1 -> A_3;
}
''')
        env.max_jobs = 3
        st = time.time()
        MP_Executor(wf).run()
        self.assertLess(time.time() - st, 5)
        for f in ['A1.txt']:
            self.assertTrue(FileTarget(f).exists())
            FileTarget(f).remove('both')
Example #9
0
    def testSoSRun(self):
        '''Test action sos_run with keyword parameters'''
        for f in ['0.txt', '1.txt']:
            FileTarget(f).remove('both')
        script = SoS_Script(r'''
[A]
parameter: num=5
sh:
    touch ${num}.txt

[batch]
for k in range(2):
    sos_run('A', num=k)
''')
        env.verbosity = 3
        wf = script.workflow('batch')
        Base_Executor(wf).run()
        for f in ['0.txt', '1.txt']:
            self.assertTrue(FileTarget(f).exists())
            FileTarget(f).remove('both')
Example #10
0
    def testArgs(self):
        '''Test args option of scripts'''
        FileTarget('a.txt').remove('both')
        script = SoS_Script(r'''
[0]
sh: args='-n'
    touch a.txt
''')
        wf = script.workflow()
        Base_Executor(wf).run()
        self.assertFalse(os.path.exists('a.txt'))
Example #11
0
File: test_dag.py Project: mr-c/SOS
    def testReverseSharedVariable(self):
        '''Test shared variables defined in auxiliary steps'''
        FileTarget('a.txt').remove('both')
        script = SoS_Script(r'''
[A: shared='b', provides='a.txt']
b = 1
sh:
    touch a.txt

[B_1]
depends: 'a.txt'

[B_2]
print(b)

''')
        wf = script.workflow('B')
        Base_Executor(wf).run()
        self.assertTrue(env.sos_dict['b'], 1)
Example #12
0
    def testPandoc(self):
        '''Test action pandoc'''
        if not shutil.which('pandoc'):
            return
        script = SoS_Script(r'''
[10]

report:
## Some random figure

Generated by matplotlib


[100]
# generate report
output: 'myreport.html'
pandoc(output=_output[0], to='html')
''')
        wf = script.workflow()
        Base_Executor(wf).run()
        self.assertTrue(os.path.isfile('myreport.html'))
        #
        FileTarget('myreport.html').remove('both')
Example #13
0
    def testIncludedNestedWorkFlow(self):
        '''Test the source option of sos_run'''
        # nested subworkflow with step option and others
        self.touch(['a.txt', 'b.txt'])
        #
        shutil.rmtree('.sos')
        os.makedirs('.sos/.runtime')
        with open('inc.sos', 'w') as sos:
            sos.write('''
# test sos script

# global definition
GLB = 5
parameter: parB = 10

[A_1: shared='executed']
executed.append('t.' + step_name)
output: _input[0] + '.a1'
sh:
    touch ${output}

[A_2: shared='executed']
executed.append('t.' + step_name)
output: _input[0] + '.a2'
sh:
    touch ${output}
''')
        script = SoS_Script('''
%from inc include *

if 'executed' not in locals():
    executed = []

[b_1: skip=False, shared='executed']
executed.append(step_name)
input: 'a.txt', 'b.txt', group_by='single'
sos_run('A')
''')
        wf = script.workflow('b')
        Base_Executor(wf).run()
        self.assertEqual(env.sos_dict['GLB'], 5)
        self.assertEqual(env.sos_dict['parB'], 10)
        self.assertEqual(env.sos_dict['executed'], ['b_1', 't.A_1', 't.A_2', 't.A_1', 't.A_2'])
        #
        shutil.rmtree('.sos')
        os.makedirs('.sos/.runtime')
        for file in ('a.txt.a1', 'a.txt.a1.a2', 'b.txt.a1', 'b.txt.a1.a2'):
            FileTarget(file).remove('both')
        script = SoS_Script('''
%include inc as k

if 'executed' not in locals():
    executed = []

[b_1: skip=False, shared='executed']
executed.append('g.' + step_name)
input: 'a.txt', 'b.txt', group_by='single'
sos_run('k.A')
''')
        wf = script.workflow('b')
        Base_Executor(wf).run()
        self.assertEqual(env.sos_dict['k'].GLB, 5)
        self.assertEqual(env.sos_dict['k'].parB, 10)
        self.assertEqual(env.sos_dict['executed'], ['g.b_1', 't.k.A_1', 't.k.A_2', 't.k.A_1', 't.k.A_2'])
        #
        os.remove('inc.sos')
Example #14
0
 def tearDown(self):
     #
     for f in self.temp_files:
         FileTarget(f).remove('both')
     subprocess.call('redis-cli shutdown', shell=True)
Example #15
0
    def testNestedWorkflow(self):
        '''Test the creation and execution of combined workfow'''
        self.touch(['a.txt', 'b.txt', 'b.begin'])
        script = SoS_Script('''
if 'executed' not in locals():
    executed = []
if 'inputs' not in locals():
    inputs = []

[a_1: shared=['executed', 'inputs']]
executed.append(step_name)
inputs.append(input)
[a_2: shared=['executed', 'inputs']]
executed.append(step_name)
inputs.append(input)
[a_3: shared=['executed', 'inputs']]
executed.append(step_name)
inputs.append(input)
[a_4: shared=['executed', 'inputs']]
executed.append(step_name)
output: 'a.done'
inputs.append(input)
sh:
    touch ${output}
[b_1: shared=['executed', 'inputs']]
executed.append(step_name)
input: 'b.begin'
inputs.append(input)
[b_2: shared=['executed', 'inputs']]
executed.append(step_name)
inputs.append(input)
[b_3: shared=['executed', 'inputs']]
executed.append(step_name)
inputs.append(input)
[b_4: shared=['executed', 'inputs']]
executed.append(step_name)
output: 'b.txt'
inputs.append(input)
[c: shared=['executed', 'inputs']]
executed.append(step_name)
input: 'a.txt'
output: 'b.txt'
inputs.append(input)
sos_run('a+b')
''')
        env.sig_mode = 'ignore'
        wf = script.workflow('c')
        Base_Executor(wf).run()
        # order of execution is not guaranteed
        self.assertEqual(sorted(env.sos_dict['executed']), sorted(['c_0', 'a_1', 'a_2', 'a_3', 'a_4',
            'b_1', 'b_2', 'b_3', 'b_4']))
        # step will be looped
        self.touch(['a.txt', 'b.txt'])
        script = SoS_Script('''
if 'executed' not in locals():
    executed = []
if 'inputs' not in locals():
    inputs = []

[a_1:shared=['executed', 'inputs']]
executed.append(step_name)
output: _input[0] + '.a1'
inputs.append(input)
sh:
    touch ${output}
[a_2:shared=['executed', 'inputs']]
executed.append(step_name)
output: _input[0] + '.a2'
inputs.append(input)
sh:
    touch ${output}
[c:shared=['executed', 'inputs']]
executed.append(step_name)
input: 'a.txt', 'b.txt', group_by='single'
inputs.append(_input)
sos_run('a')
''')
        wf = script.workflow('c')
        Base_Executor(wf).run()
        self.assertEqual(env.sos_dict['executed'], ['c_0', 'a_1', 'a_2', 'a_1', 'a_2'])
        #self.assertEqual(env.sos_dict['inputs'], [['a.txt'], ['a.txt'], ['a.txt.a1'], ['b.txt'], ['b.txt'], ['b.txt.a1']])
        for file in ('a.txt.a1', 'a.txt.a1.a2', 'b.txt.a1', 'b.txt.a1.a2'):
            FileTarget(file).remove('both')
        #
        # allow specifying a single step
        # step will be looped
        script = SoS_Script('''
if 'executed' not in locals():
    executed = []
[a_1:shared='executed']
executed.append(step_name)
[a_2:shared='executed']
executed.append(step_name)
[c_0:shared='executed']
executed.append(step_name)
[c_1:shared='executed']
executed.append(step_name)
input: 'a.txt', 'b.txt', group_by='single'
sos_run('a:2')
''')
        wf = script.workflow('c')
        Base_Executor(wf).run()
        self.assertEqual(env.sos_dict['executed'], ['c_0', 'c_1', 'a_2',  'a_2'])
        # allow specifying a single step
        # step will be looped
        script = SoS_Script('''
if 'executed' not in locals():
    executed = []
[a_1:shared='executed']
executed.append(step_name)
[a_2:shared='executed']
executed.append(step_name)
[c_0:shared='executed']
executed.append(step_name)
[c_1:shared='executed']
executed.append(step_name)
input: 'a.txt', 'b.txt', group_by='single'
sos_run('a:2')
''')
        wf = script.workflow('c')
        Base_Executor(wf).run()
        self.assertEqual(env.sos_dict['executed'], ['c_0', 'c_1', 'a_2', 'a_2'])
        #
        # recursive subworkflow not allowed
        script = SoS_Script('''
if 'executed' not in locals():
    executed = []
[a_1:shared='executed']
executed.append(step_name)
[a_2:shared='executed']
executed.append(step_name)
[c_0:shared='executed']
executed.append(step_name)
[c_1:shared='executed']
executed.append(step_name)
input: 'a.txt', 'b.txt', group_by='single'
sos_run('a_2+c')
''')
        wf = script.workflow('c')
        self.assertRaises(ExecuteError, Base_Executor(wf).run)
        #
        # nested subworkflow is allowed
        script = SoS_Script('''
if 'executed' not in locals():
    executed = []
[a_1:shared='executed']
executed.append(step_name)
[a_2:shared='executed']
executed.append(step_name)
[a_3:shared='executed']
executed.append(step_name)
[b_1:shared='executed']
executed.append(step_name)
[b_2:shared='executed']
executed.append(step_name)
sos_run('a:1-2')
[c_0:shared='executed']
executed.append(step_name)
[c_1:shared='executed']
executed.append(step_name)
input: 'a.txt'
sos_run('a+b')
''')
        wf = script.workflow('c')
        Base_Executor(wf).run()
        self.assertEqual(env.sos_dict['executed'], ['c_0', 'c_1', 'a_1', 'a_2', 'a_3',
            'b_1', 'b_2', 'a_1', 'a_2'])
        #
        #
        # nested subworkflow with step option and others
        script = SoS_Script('''
if 'executed' not in locals():
    executed = []
[a_1:shared='executed']
executed.append(step_name)
[a_2:shared='executed']
executed.append(step_name)
[a_3:shared='executed']
executed.append(step_name)
[b:shared='executed']
executed.append(step_name)
input: 'a.txt', 'b.txt', group_by='single'
sos_run('a:3+a:1')
[d:shared='executed']
executed.append(step_name)
input: 'a.txt', 'b.txt', group_by='single'
sos_run('a:2')
[e2_2:shared='executed']
executed.append(step_name)
input: 'a.txt', 'b.txt', group_by='single'
''')
        wf = script.workflow('b')
        Base_Executor(wf).run()
        self.assertEqual(env.sos_dict['executed'], ['b_0', 'a_3', 'a_1', 'a_3', 'a_1'])
        wf = script.workflow('d')
        Base_Executor(wf).run()
        self.assertEqual(env.sos_dict['executed'], ['d_0', 'a_2', 'a_2'])
        wf = script.workflow('e2')
        Base_Executor(wf).run()
        self.assertEqual(env.sos_dict['executed'], ['e2_2'])
        #
        # clean up
        FileTarget('a.done').remove('both')
Example #16
0
File: test_dag.py Project: mr-c/SOS
    def testLongChain(self):
        '''Test long make file style dependencies.'''
        #
        for f in [
                'A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt',
                'C1.txt', 'C3.txt', 'C4.txt'
        ]:
            FileTarget(f).remove('both')
        #
        #  A1 <- B1 <- B2 <- B3
        #   |
        #   |
        #  \/
        #  A2 <- B2 <- C1 <- C2 <- C4
        #                    C3
        #
        script = SoS_Script('''
[A_1]
input: 'B1.txt'
output: 'A1.txt'
sh:
    touch A1.txt

[A_2]
depends:  'B2.txt'
sh:
    touch A2.txt

[B1: provides='B1.txt']
depends: 'B2.txt'
sh:
    touch B1.txt

[B2: provides='B2.txt']
depends: 'B3.txt', 'C1.txt'
sh:
    touch B2.txt

[B3: provides='B3.txt']
sh:
    touch B3.txt

[C1: provides='C1.txt']
depends: 'C2.txt', 'C3.txt'
sh:
    touch C1.txt

[C2: provides='C2.txt']
depends: 'C4.txt'
sh:
    touch C2.txt

[C3: provides='C3.txt']
depends: 'C4.txt'
sh:
    touch C3.txt

[C4: provides='C4.txt']
sh:
    touch C4.txt

        ''')
        # the workflow should call step K for step C_2, but not C_3
        wf = script.workflow()
        dag = Base_Executor(wf).initialize_dag()
        self.assertDAG(
            dag, '''
strict digraph "" {
"C4 ['C4.txt']";
"B1 ['B1.txt']";
"C1 ['C1.txt']";
"C2 ['C2.txt']";
"C3 ['C3.txt']";
A_1;
"B2 ['B2.txt']";
"B3 ['B3.txt']";
A_2;
"C4 ['C4.txt']" -> "C2 ['C2.txt']";
"C4 ['C4.txt']" -> "C3 ['C3.txt']";
"B1 ['B1.txt']" -> A_1;
"C1 ['C1.txt']" -> "B2 ['B2.txt']";
"C2 ['C2.txt']" -> "C1 ['C1.txt']";
"C3 ['C3.txt']" -> "C1 ['C1.txt']";
A_1 -> A_2;
"B2 ['B2.txt']" -> "B1 ['B1.txt']";
"B2 ['B2.txt']" -> A_2;
"B3 ['B3.txt']" -> "B2 ['B2.txt']";
}
''')
        Base_Executor(wf).run()
        for f in [
                'A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt',
                'C1.txt', 'C3.txt', 'C4.txt'
        ]:
            t = FileTarget(f)
            self.assertTrue(t.exists())
            t.remove('both')
Example #17
0
    def testTarget(self):
        '''Test executing only part of a workflow.'''
        #
        for f in ['A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']:
            FileTarget(f).remove('both')
        #
        #  A1 <- B1 <- B2 <- B3
        #   |
        #   |
        #  \/
        #  A2 <- B2 <- C1 <- C2 <- C4
        #                    C3
        #
        script = SoS_Script('''
[A_1]
input: 'B1.txt'
output: 'A1.txt'
sh:
    touch A1.txt

[A_2]
depends:  'B2.txt'
sh:
    touch A2.txt

[B1: provides='B1.txt']
depends: 'B2.txt'
sh:
    touch B1.txt

[B2: provides='B2.txt']
depends: 'B3.txt', 'C1.txt'
sh:
    touch B2.txt

[B3: provides='B3.txt']
sh:
    touch B3.txt

[C1: provides='C1.txt']
depends: 'C2.txt', 'C3.txt'
sh:
    touch C1.txt

[C2: provides='C2.txt']
depends: 'C4.txt'
sh:
    touch C2.txt

[C3: provides='C3.txt']
depends: 'C4.txt'
sh:
    touch C3.txt

[C4: provides='C4.txt']
sh:
    touch C4.txt

        ''')
        # the workflow should call step K for step C_2, but not C_3
        wf = script.workflow()
        #
        # test 1, we only need to generate target 'B1.txt'
        dag = Base_Executor(wf).initialize_dag(targets=['B1.txt'])
        # note that A2 is no longer mentioned
        self.assertDAG(dag,
'''
strict digraph "" {
"B3 ['B3.txt']";
"C4 ['C4.txt']";
"C2 ['C2.txt']";
"C1 ['C1.txt']";
"B1 ['B1.txt']";
"B2 ['B2.txt']";
"C3 ['C3.txt']";
"B3 ['B3.txt']" -> "B2 ['B2.txt']";
"C4 ['C4.txt']" -> "C3 ['C3.txt']";
"C4 ['C4.txt']" -> "C2 ['C2.txt']";
"C2 ['C2.txt']" -> "C1 ['C1.txt']";
"C1 ['C1.txt']" -> "B2 ['B2.txt']";
"B2 ['B2.txt']" -> "B1 ['B1.txt']";
"C3 ['C3.txt']" -> "C1 ['C1.txt']";
}
''')
        Base_Executor(wf).run(targets=['B1.txt'])
        for f in ['A1.txt', 'A2.txt']:
            self.assertFalse(FileTarget(f).exists())
        for f in ['C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']:
            t = FileTarget(f)
            self.assertTrue(t.exists())
            t.remove('both')
        #
        # test 2, we would like to generate two files
        dag = Base_Executor(wf).initialize_dag(targets=['B2.txt', 'C2.txt'])
        # note that A2 is no longer mentioned
        self.assertDAG(dag,
'''
strict digraph "" {
"C4 ['C4.txt']";
"B2 ['B2.txt']";
"C3 ['C3.txt']";
"B3 ['B3.txt']";
"C2 ['C2.txt']";
"C1 ['C1.txt']";
"C4 ['C4.txt']" -> "C2 ['C2.txt']";
"C4 ['C4.txt']" -> "C3 ['C3.txt']";
"C3 ['C3.txt']" -> "C1 ['C1.txt']";
"B3 ['B3.txt']" -> "B2 ['B2.txt']";
"C2 ['C2.txt']" -> "C1 ['C1.txt']";
"C1 ['C1.txt']" -> "B2 ['B2.txt']";
}
''')
        Base_Executor(wf).run(targets=['B2.txt', 'C2.txt'])
        for f in ['A1.txt', 'B1.txt', 'A2.txt']:
            self.assertFalse(FileTarget(f).exists())
        for f in ['C2.txt', 'B2.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']:
            t = FileTarget(f)
            self.assertTrue(t.exists())
            t.remove('both')
        #
        # test 3, generate two separate trees
        #
        dag = Base_Executor(wf).initialize_dag(targets=['B3.txt', 'C2.txt'])
        # note that A2 is no longer mentioned
        self.assertDAG(dag,
'''
strict digraph "" {
"B3 ['B3.txt']";
"C2 ['C2.txt']";
"C4 ['C4.txt']";
"C4 ['C4.txt']" -> "C2 ['C2.txt']";
}
''')
        Base_Executor(wf).run(targets=['B3.txt', 'C2.txt'])
        for f in ['A1.txt', 'B1.txt', 'A2.txt', 'B2.txt', 'C1.txt', 'C3.txt']:
            self.assertFalse(FileTarget(f).exists())
        for f in ['C2.txt', 'B3.txt', 'C4.txt']:
            t = FileTarget(f)
            self.assertTrue(t.exists())
            t.remove('both')
Example #18
0
 def tearDown(self):
     for f in self.temp_files:
         FileTarget(f).remove('both')
Example #19
0
File: test_dag.py Project: mr-c/SOS
    def testTarget(self):
        '''Test executing only part of a workflow.'''
        #
        for f in [
                'A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt',
                'C1.txt', 'C3.txt', 'C4.txt'
        ]:
            FileTarget(f).remove('both')
        #
        #  A1 <- B1 <- B2 <- B3
        #   |
        #   |
        #  \/
        #  A2 <- B2 <- C1 <- C2 <- C4
        #                    C3
        #
        script = SoS_Script('''
[A_1]
input: 'B1.txt'
output: 'A1.txt'
sh:
    touch A1.txt

[A_2]
depends:  'B2.txt'
sh:
    touch A2.txt

[B1: provides='B1.txt']
depends: 'B2.txt'
sh:
    touch B1.txt

[B2: provides='B2.txt']
depends: 'B3.txt', 'C1.txt'
sh:
    touch B2.txt

[B3: provides='B3.txt']
sh:
    touch B3.txt

[C1: provides='C1.txt']
depends: 'C2.txt', 'C3.txt'
sh:
    touch C1.txt

[C2: provides='C2.txt']
depends: 'C4.txt'
sh:
    touch C2.txt

[C3: provides='C3.txt']
depends: 'C4.txt'
sh:
    touch C3.txt

[C4: provides='C4.txt']
sh:
    touch C4.txt

        ''')
        # the workflow should call step K for step C_2, but not C_3
        wf = script.workflow()
        #
        # test 1, we only need to generate target 'B1.txt'
        dag = Base_Executor(wf).initialize_dag(targets=['B1.txt'])
        # note that A2 is no longer mentioned
        self.assertDAG(
            dag, '''
strict digraph "" {
"B3 ['B3.txt']";
"C4 ['C4.txt']";
"C2 ['C2.txt']";
"C1 ['C1.txt']";
"B1 ['B1.txt']";
"B2 ['B2.txt']";
"C3 ['C3.txt']";
"B3 ['B3.txt']" -> "B2 ['B2.txt']";
"C4 ['C4.txt']" -> "C3 ['C3.txt']";
"C4 ['C4.txt']" -> "C2 ['C2.txt']";
"C2 ['C2.txt']" -> "C1 ['C1.txt']";
"C1 ['C1.txt']" -> "B2 ['B2.txt']";
"B2 ['B2.txt']" -> "B1 ['B1.txt']";
"C3 ['C3.txt']" -> "C1 ['C1.txt']";
}
''')
        Base_Executor(wf).run(targets=['B1.txt'])
        for f in ['A1.txt', 'A2.txt']:
            self.assertFalse(FileTarget(f).exists())
        for f in [
                'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt',
                'C4.txt'
        ]:
            t = FileTarget(f)
            self.assertTrue(t.exists())
            t.remove('both')
        #
        # test 2, we would like to generate two files
        dag = Base_Executor(wf).initialize_dag(targets=['B2.txt', 'C2.txt'])
        # note that A2 is no longer mentioned
        self.assertDAG(
            dag, '''
strict digraph "" {
"C4 ['C4.txt']";
"B2 ['B2.txt']";
"C3 ['C3.txt']";
"B3 ['B3.txt']";
"C2 ['C2.txt']";
"C1 ['C1.txt']";
"C4 ['C4.txt']" -> "C2 ['C2.txt']";
"C4 ['C4.txt']" -> "C3 ['C3.txt']";
"C3 ['C3.txt']" -> "C1 ['C1.txt']";
"B3 ['B3.txt']" -> "B2 ['B2.txt']";
"C2 ['C2.txt']" -> "C1 ['C1.txt']";
"C1 ['C1.txt']" -> "B2 ['B2.txt']";
}
''')
        Base_Executor(wf).run(targets=['B2.txt', 'C2.txt'])
        for f in ['A1.txt', 'B1.txt', 'A2.txt']:
            self.assertFalse(FileTarget(f).exists())
        for f in ['C2.txt', 'B2.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']:
            t = FileTarget(f)
            self.assertTrue(t.exists())
            t.remove('both')
        #
        # test 3, generate two separate trees
        #
        dag = Base_Executor(wf).initialize_dag(targets=['B3.txt', 'C2.txt'])
        # note that A2 is no longer mentioned
        self.assertDAG(
            dag, '''
strict digraph "" {
"B3 ['B3.txt']";
"C2 ['C2.txt']";
"C4 ['C4.txt']";
"C4 ['C4.txt']" -> "C2 ['C2.txt']";
}
''')
        Base_Executor(wf).run(targets=['B3.txt', 'C2.txt'])
        for f in ['A1.txt', 'B1.txt', 'A2.txt', 'B2.txt', 'C1.txt', 'C3.txt']:
            self.assertFalse(FileTarget(f).exists())
        for f in ['C2.txt', 'B3.txt', 'C4.txt']:
            t = FileTarget(f)
            self.assertTrue(t.exists())
            t.remove('both')
Example #20
0
File: test_dag.py Project: mr-c/SOS
    def testPatternReuse(self):
        '''Test repeated use of steps that use pattern and produce different files.'''
        #
        for f in [
                'A1.txt', 'A2.txt', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p'
        ]:
            FileTarget(f).remove('both')
        #
        #  A1 <- P <- B1
        #  A1 <- P <- B2
        #  A2
        #
        script = SoS_Script('''
[A_1]
input: 'B1.txt.p', 'B2.txt.p'
output: 'A1.txt'
sh:
    touch A1.txt

[A_2]
sh:
    touch A2.txt

[B1: provides='B1.txt']
sh:
    touch B1.txt

[B2: provides='B2.txt']
sh:
    touch B2.txt

[P: provides='{filename}.p']
input: filename
sh:
    touch ${output}
''')
        # the workflow should call step K for step C_2, but not C_3
        wf = script.workflow()
        dag = Base_Executor(wf).initialize_dag()
        self.assertDAG(
            dag, '''
strict digraph "" {
"P ['B2.txt.p']";
"B1 ['B1.txt']";
"B2 ['B2.txt']";
A_2;
A_1;
"P ['B1.txt.p']";
"P ['B2.txt.p']" -> A_1;
"B1 ['B1.txt']" -> "P ['B1.txt.p']";
"B2 ['B2.txt']" -> "P ['B2.txt.p']";
A_1 -> A_2;
"P ['B1.txt.p']" -> A_1;
}
''')
        Base_Executor(wf).run()
        for f in [
                'A1.txt', 'A2.txt', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p'
        ]:
            t = FileTarget(f)
            self.assertTrue(t.exists())
            t.remove('both')
Example #21
0
 def tearDown(self):
     for f in self.temp_files:
         FileTarget(f).remove('both')
     os.chdir(self.olddir)
Example #22
0
    def testLongChain(self):
        '''Test long make file style dependencies.'''
        #
        for f in ['A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']:
            FileTarget(f).remove('both')
        #
        #  A1 <- B1 <- B2 <- B3
        #   |
        #   |
        #  \/
        #  A2 <- B2 <- C1 <- C2 <- C4
        #                    C3
        #
        script = SoS_Script('''
[A_1]
input: 'B1.txt'
output: 'A1.txt'
sh:
    touch A1.txt

[A_2]
depends:  'B2.txt'
sh:
    touch A2.txt

[B1: provides='B1.txt']
depends: 'B2.txt'
sh:
    touch B1.txt

[B2: provides='B2.txt']
depends: 'B3.txt', 'C1.txt'
sh:
    touch B2.txt

[B3: provides='B3.txt']
sh:
    touch B3.txt

[C1: provides='C1.txt']
depends: 'C2.txt', 'C3.txt'
sh:
    touch C1.txt

[C2: provides='C2.txt']
depends: 'C4.txt'
sh:
    touch C2.txt

[C3: provides='C3.txt']
depends: 'C4.txt'
sh:
    touch C3.txt

[C4: provides='C4.txt']
sh:
    touch C4.txt

        ''')
        # the workflow should call step K for step C_2, but not C_3
        wf = script.workflow()
        dag = Base_Executor(wf).initialize_dag()
        self.assertDAG(dag,
'''
strict digraph "" {
"C4 ['C4.txt']";
"B1 ['B1.txt']";
"C1 ['C1.txt']";
"C2 ['C2.txt']";
"C3 ['C3.txt']";
A_1;
"B2 ['B2.txt']";
"B3 ['B3.txt']";
A_2;
"C4 ['C4.txt']" -> "C2 ['C2.txt']";
"C4 ['C4.txt']" -> "C3 ['C3.txt']";
"B1 ['B1.txt']" -> A_1;
"C1 ['C1.txt']" -> "B2 ['B2.txt']";
"C2 ['C2.txt']" -> "C1 ['C1.txt']";
"C3 ['C3.txt']" -> "C1 ['C1.txt']";
A_1 -> A_2;
"B2 ['B2.txt']" -> "B1 ['B1.txt']";
"B2 ['B2.txt']" -> A_2;
"B3 ['B3.txt']" -> "B2 ['B2.txt']";
}
''')
        Base_Executor(wf).run()
        for f in ['A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']:
            t = FileTarget(f)
            self.assertTrue(t.exists())
            t.remove('both')