def testSoSAction(self): '''Test sos_action decorator''' script = SoS_Script(r""" from sos.actions import SoS_Action @SoS_Action(run_mode='run') def func_run(): return 1 @SoS_Action(run_mode=['run', 'dryrun']) def func_both(): return 1 [0: shared=('b', 'c')] b=func_run() c=func_both() """) wf = script.workflow() Base_Executor(wf).dryrun() self.assertTrue(isinstance(env.sos_dict['b'], Undetermined)) self.assertTrue(isinstance(env.sos_dict['c'], Undetermined)) # Base_Executor(wf).run() self.assertEqual(env.sos_dict['b'], 1) self.assertEqual(env.sos_dict['c'], 1)
def testVarOutput(self): '''Test early appearance of variable output''' script = SoS_Script(''' [0] seq = range(3) input: for_each='seq' output: "test${_seq}.txt" print(output) ''') wf = script.workflow() # this does not work before until we make variable output available sooner Base_Executor(wf).dryrun() # however, output should be the same in task script = SoS_Script(''' [0] seq = range(3) input: for_each='seq' output: "test${_seq}.txt" assert(len(output), _index + 1) task: assert(len(output), 3) ''') wf = script.workflow() # this does not work before until we make variable output available sooner Base_Executor(wf).dryrun()
def testDynamicNestedWorkflow(self): '''Test nested workflow controlled by command line option''' script = SoS_Script(''' if 'executed' not in locals(): executed = [] parameter: wf='a' [a_1:shared='executed'] executed.append(step_name) [a_2:shared='executed'] executed.append(step_name) [a_3:shared='executed'] executed.append(step_name) [b_1:shared='executed'] executed.append(step_name) [b_2:shared='executed'] executed.append(step_name) [b_3:shared='executed'] executed.append(step_name) [default:shared='executed'] executed.append(step_name) sos_run(wf) ''') wf = script.workflow() Base_Executor(wf, args=['--wf', 'b']).run() self.assertEqual(env.sos_dict['executed'], [ 'default_0', 'b_1', 'b_2', 'b_3']) # Base_Executor(wf, args=['--wf', 'a']).run() self.assertEqual(env.sos_dict['executed'], ['default_0', 'a_1', 'a_2', 'a_3'])
def testSections(self): '''Test section definitions''' # bad names for badname in ['56_1', '_a', 'a_', '1x', '*', '?']: self.assertRaises(ParsingError, SoS_Script, '[{}]'.format(badname)) # bad options for badoption in ['ss', 'skip a', 'skip:_', 'skip, skip']: self.assertRaises(ParsingError, SoS_Script, '[0:{}]'.format(badoption)) # option value should be a valid python expression for badoption in ['sigil=a', 'sigil="[]"', 'sigil="| |"']: self.assertRaises((ValueError, ParsingError), SoS_Script, '[0:{}]'.format(badoption)) # good options for goodoption in ['sigil="[ ]"', 'shared="a"']: SoS_Script('[0:{}]'.format(goodoption)) # allowed names for name in ['a5', 'a_5', '*_0', 'a*1_100']: SoS_Script('[{}]'.format(name)) # allowed names with alias for name in ['a5 (p1)', 'a_5 (something fun)', '*_0 (no way)', 'a*1_100']: SoS_Script('[{}]'.format(name)) # no directive in global section self.assertRaises(ParsingError, SoS_Script, '''input: 'filename' ''') # duplicate sections self.assertRaises(ParsingError, SoS_Script, '''[1]\n[1]''') self.assertRaises(ParsingError, SoS_Script, '''[1]\n[3]\n[2,1]''') self.assertRaises(ParsingError, SoS_Script, '''[a_1]\n[a_3]\n[*_1]''') # # no duplicated section header SoS_Script('''[a_1]\n[a_3]\n[b*_1]''')
def testTextRepr(self): # the " as the last character can lead to problems... script = SoS_Script( ''' run: echo "Hi, This is from bash"''' ) wf = script.workflow() Base_Executor(wf).run() # for text in ('"""a"""', '"b"', r'"""\na\\nb"""', r"'''a\nb'''", """ "a'\\"='" """): script = SoS_Script( r''' a = 1 python: with open('tmp.txt', 'w') as tmp: tmp.write({} + '{}') k = """b"""'''.format( text, "${a}" ) ) wf = script.workflow() Base_Executor(wf).run() with open("tmp.txt") as tmp: self.assertEqual(tmp.read(), eval(text) + "1") os.remove("tmp.txt")
def testChainedDepends(self): '''Test chain dependent''' shutil.rmtree('.sos') os.makedirs('.sos/.runtime') script = SoS_Script(r''' # this step provides variable `var` [index: provides='{filename}.bam.bai'] input: "${filename}.bam" sh: echo "Generating ${output}" touch ${output} [call: provides='{filename}.vcf'] input: "${filename}.bam" depends: "${input}.bai" sh: echo "Calling variants from ${input} with ${depends} to ${output}" touch ${output} ''') FileTarget('a.bam.bai').remove('both') FileTarget('a.vcf').remove('both') self.touch('a.bam') Base_Executor(script.workflow()).run(targets=['a.vcf']) for file in ('a.vcf', 'a.bam', 'a.bam.bai'): FileTarget(file).remove('both')
def testGlobalVariables(self): '''Test definition of variables''' # global section cannot have directive self.assertRaises(ParsingError, SoS_Script, '''input: 'filename' ''') # allow definition SoS_Script('''a = '1' ''') SoS_Script('''a = ['a', 'b'] ''') # but this one has incorrect syntax self.assertRaises(ParsingError, SoS_Script, '''a = 'b ''') # This one also does not work because b is not defined. #delattr(env, 'sos_dict') #script = SoS_Script('''a = b\n[0] ''') #wf = script.workflow() #dag = Base_Executor(wf).dryrun() #self.assertRaises(ValueError, Base_Executor(wf).run, dag) # multi-line string literal SoS_Script('''a = """ this is a multi line string """ ''') # multi-line list literal, even with newline in between SoS_Script('''a = [ 'a', 'b' ] ''') # SoS_Script(filename='scripts/section1.sos')
def testOverwriteKeyword(self): '''Test overwrite sos keyword with user defined one.''' FileTarget('a.txt').remove('both') # script = SoS_Script(''' def run(script): pass [1] run: touch a.txt ''') wf = script.workflow() Base_Executor(wf).run() self.assertFalse(os.path.isfile('a.txt')) # script = SoS_Script(''' parameter: run = 5 [1] run: touch a.txt ''') wf = script.workflow() self.assertRaises(Exception, Base_Executor(wf).run)
def testInput(self): '''Test input directive''' self.touch(['a.txt', 'b.txt', 'a.pdf', 'a0', 'a1']) script = SoS_Script(''' [0] files = ['a.txt', 'b.txt'] input: 'a.pdf', files ''') wf = script.workflow('default') Base_Executor(wf).dryrun() # # test input types script = SoS_Script(''' [0:shared={'i':'input', 'o':'output'}] files = ("a${i}" for i in range(2)) input: {'a.txt', 'b.txt'}, files output: ("a${x}" for x in _input) ''') wf = script.workflow() Base_Executor(wf).dryrun() self.assertEqual(sorted(env.sos_dict['i']), ['a.txt', 'a0', 'a1', 'b.txt']) self.assertEqual(sorted(env.sos_dict['o']), ['aa.txt', 'aa0', 'aa1', 'ab.txt'])
def testInclude(self): '''Test include keyword''' with open('inc.sos', 'w') as ts: ts.write(''' # a slave script to be included gv = 1 [A_1] [A_2] [B] ''') script = SoS_Script(''' %include inc res = inc.gv [0] ''') wf = script.workflow() Base_Executor(wf).dryrun() Base_Executor(wf).run() self.assertEqual(env.sos_dict['res'], 1) # # include with alias script = SoS_Script(''' %include inc as tt res1 = tt.gv [0] ''') wf = script.workflow() Base_Executor(wf).dryrun() Base_Executor(wf).run() self.assertEqual(env.sos_dict['res1'], 1) os.remove('inc.sos')
def testFromInclude(self): '''Test include keyword''' with open('inc.sos', 'w') as ts: ts.write(''' # a slave script to be included gv = 1 [A_1] [A_2] [B] ''') script = SoS_Script(''' %from inc include gv [0] ''') wf = script.workflow() Base_Executor(wf).dryrun() Base_Executor(wf).run() self.assertEqual(env.sos_dict['gv'], 1) # # include with alias script = SoS_Script(''' %from inc include gv as g res1 = g [0] ''') wf = script.workflow() Base_Executor(wf).dryrun() Base_Executor(wf).run() self.assertEqual(env.sos_dict['res1'], 1)
def testSharedVar(self): '''Test shared var with rq queue''' script = SoS_Script( ''' [work_1: shared = {'data': 'output'}] input: "1.txt", "2.txt", group_by = 'single', pattern = '{name}.{ext}' output: expand_pattern('{_name}.out') task: concurrent = True run: touch ${_output} [work_2] input: "1.txt", "2.txt", group_by = 'single', pattern = '{name}.{ext}', paired_with = ['data'] output: expand_pattern('{_name}.out2') task: concurrent = True run: touch ${_data} ${_output} [default] sos_run("work:1+work:2") ''') self.touch(['1.txt', '2.txt']) subprocess.call('sos remove . -t -y', shell=True) wf = script.workflow() RQ_Executor(wf).run() for f in ['1.out', '1.out2', '2.out', '2.out2']: self.assertTrue(FileTarget(f).exists('target')) FileTarget(f).remove('both')
def testSharedVar(self): '''Test shared var with rq queue''' script = SoS_Script(''' [work_1: shared = {'data': 'output'}] input: "1.txt", "2.txt", group_by = 'single', pattern = '{name}.{ext}' output: expand_pattern('{_name}.out') task: concurrent = True run: touch ${_output} [work_2] input: "1.txt", "2.txt", group_by = 'single', pattern = '{name}.{ext}', paired_with = ['data'] output: expand_pattern('{_name}.out2') task: concurrent = True run: touch ${_data} ${_output} [default] sos_run("work:1+work:2") ''') self.touch(['1.txt', '2.txt']) subprocess.call('sos remove . -t -y', shell=True) wf = script.workflow() RQ_Executor(wf).run() for f in ['1.out', '1.out2', '2.out', '2.out2']: self.assertTrue(FileTarget(f).exists('target')) FileTarget(f).remove('both')
def testPatternReuse(self): '''Test repeated use of steps that use pattern and produce different files.''' # for f in ['A1.txt', 'A2.txt', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p']: FileTarget(f).remove('both') # # A1 <- P <- B1 # A1 <- P <- B2 # A2 # script = SoS_Script(''' [A_1] input: 'B1.txt.p', 'B2.txt.p' output: 'A1.txt' sh: touch A1.txt [A_2] sh: touch A2.txt [B1: provides='B1.txt'] sh: touch B1.txt [B2: provides='B2.txt'] sh: touch B2.txt [P: provides='{filename}.p'] input: filename sh: touch ${output} ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow() dag = Base_Executor(wf).initialize_dag() self.assertDAG(dag, ''' strict digraph "" { "P ['B2.txt.p']"; "B1 ['B1.txt']"; "B2 ['B2.txt']"; A_2; A_1; "P ['B1.txt.p']"; "P ['B2.txt.p']" -> A_1; "B1 ['B1.txt']" -> "P ['B1.txt.p']"; "B2 ['B2.txt']" -> "P ['B2.txt.p']"; A_1 -> A_2; "P ['B1.txt.p']" -> A_1; } ''') Base_Executor(wf).run() for f in ['A1.txt', 'A2.txt', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p']: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both')
def testBashInDocker(self): '''Test action bash in docker environment''' script = SoS_Script(r''' [0] bash: docker_image='ubuntu' echo 'Echo' ''') wf = script.workflow() Base_Executor(wf).run()
def testYAMLConfig(self): '''Test config file in yaml format''' with open('myconfig.yml', 'w') as config: config.write(''' # Lines beginning with # are skipped when the JSON is parsed, so we can # put comments into our JSON configuration files { StoreOwner : "John Doe", # List of items that we sell Fruits: [ "apple", "banana", "pear" ], Price: 1.05 } ''') with open('config.sos', 'w') as sos: sos.write(''' [0] print(CONFIG['StoreOwner']) print(CONFIG.get('StoreOwner', 'something')) print(CONFIG.get('StoreOwnerSpouse', 'someone else')) print(CONFIG.StoreOwner) ''' ) # run the command self.assertEqual(subprocess.call('sos run config.sos -c myconfig.yml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 0) # now test the value script = SoS_Script(filename='config.sos') wf = script.workflow() Base_Executor(wf, config={'config_file': 'myconfig.yml'}).run() self.assertEqual(env.sos_dict['CONFIG']['Price'], 1.05) self.assertEqual(env.sos_dict['CONFIG']['StoreOwner'], 'John Doe') self.assertEqual(env.sos_dict['CONFIG']['Fruits'], ['apple', 'banana', 'pear']) # configuration items should be readonly with open('config.sos', 'w') as sos: sos.write(''' [0] CONFIG['a'] = 'b' ''' ) # the command would fail with error message # ERROR: Failed to process statement CONFIG['a'] = 'b' # : Cannot modify a readonly dictionary. self.assertEqual(subprocess.call('sos run config.sos -c myconfig.yml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 1) # with open('config.sos', 'w') as sos: sos.write(''' [0] CONFIG.a = 'b' ''' ) # the command would fail with error message # ERROR: Failed to process statement CONFIG['a'] = 'b' # : Cannot modify a readonly dictionary. self.assertEqual(subprocess.call('sos run config.sos -c myconfig.yml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 1) # for filename in ['config.sos', 'myconfig.yml']: os.remove(filename)
def testLiteralConnection(self): '''Testing the connection of steps with by variables.''' for f in ['A1.txt']: FileTarget(f).remove('both') # # A1 introduces a shared variable ss, A3 depends on ss but not A2 # script = SoS_Script(''' [A_1: shared='p'] sh: touch 'A1.txt' p = 'A1.txt' [A_2] input: None sh: sleep 3 [A_3] input: p sh: sleep 3 [A_4] input: p sh: sleep 3 [A_5] input: dynamic(p) ''') wf = script.workflow('A') dag = Base_Executor(wf).initialize_dag() self.assertDAG(dag, ''' strict digraph "" { A_1; A_4; A_2; A_3; A_5; A_1 -> A_4; A_1 -> A_3; A_1 -> A_5; A_4 -> A_5; } ''') env.max_jobs = 3 st = time.time() MP_Executor(wf).run() self.assertLess(time.time() - st, 5) for f in ['A1.txt']: self.assertTrue(FileTarget(f).exists()) FileTarget(f).remove('both')
def testYAMLConfig(self): '''Test config file in yaml format''' with open('myconfig.yaml', 'w') as config: config.write(''' # Lines beginning with # are skipped when the JSON is parsed, so we can # put comments into our JSON configuration files { StoreOwner : "John Doe", # List of items that we sell Fruits: [ "apple", "banana", "pear" ], Price: 1.05 } ''') with open('config.sos', 'w') as sos: sos.write(''' [0] print(CONFIG['StoreOwner']) print(CONFIG.get('StoreOwner', 'something')) print(CONFIG.get('StoreOwnerSpouse', 'someone else')) print(CONFIG.StoreOwner) ''' ) # run the command self.assertEqual(subprocess.call('sos run config.sos -c myconfig.yaml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 0) # now test the value script = SoS_Script(filename='config.sos') wf = script.workflow() Base_Executor(wf, config_file='myconfig.yaml').run() self.assertEqual(env.sos_dict['CONFIG']['Price'], 1.05) self.assertEqual(env.sos_dict['CONFIG']['StoreOwner'], 'John Doe') self.assertEqual(env.sos_dict['CONFIG']['Fruits'], ['apple', 'banana', 'pear']) # configuration items should be readonly with open('config.sos', 'w') as sos: sos.write(''' [0] CONFIG['a'] = 'b' ''' ) # the command would fail with error message # ERROR: Failed to process statement CONFIG['a'] = 'b' # : Cannot modify a readonly dictionary. self.assertEqual(subprocess.call('sos run config.sos -c myconfig.yaml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 1) # with open('config.sos', 'w') as sos: sos.write(''' [0] CONFIG.a = 'b' ''' ) # the command would fail with error message # ERROR: Failed to process statement CONFIG['a'] = 'b' # : Cannot modify a readonly dictionary. self.assertEqual(subprocess.call('sos run config.sos -c myconfig.yaml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 1) # for filename in ['config.sos', 'myconfig.yaml']: os.remove(filename)
def testLiteralConnection(self): '''Testing the connection of steps with by variables.''' for f in ['A1.txt']: FileTarget(f).remove('both') # # A1 introduces a shared variable ss, A3 depends on ss but not A2 # script = SoS_Script(''' [A_1: shared='p'] sh: touch 'A1.txt' p = 'A1.txt' [A_2] input: None sh: sleep 3 [A_3] input: p sh: sleep 3 [A_4] input: p sh: sleep 3 [A_5] input: dynamic(p) ''') wf = script.workflow('A') dag = Base_Executor(wf).initialize_dag() self.assertDAG( dag, ''' strict digraph "" { A_1; A_4; A_2; A_3; A_5; A_1 -> A_4; A_1 -> A_3; A_1 -> A_5; A_4 -> A_5; } ''') env.max_jobs = 3 st = time.time() MP_Executor(wf).run() self.assertLess(time.time() - st, 5) for f in ['A1.txt']: self.assertTrue(FileTarget(f).exists()) FileTarget(f).remove('both')
def testPython3(self): script = SoS_Script(r''' [0] python3: a = {'1', '2'} print(a) ''') wf = script.workflow() Base_Executor(wf).run()
def testPythonInDocker(self): '''Test action python in docker environment''' script = SoS_Script(r''' [0] python: docker_image='python' a = {'1': 2} print(a) ''') wf = script.workflow() Base_Executor(wf).run()
def testRInDocker(self): '''Test action R in docker environment''' script = SoS_Script(r''' [0] R: docker_image='r-base' nums = rnorm(25, mean=100, sd=15) mean(nums) ''') wf = script.workflow() Base_Executor(wf).run()
def testArgs(self): '''Test args option of scripts''' FileTarget('a.txt').remove('both') script = SoS_Script(r''' [0] sh: args='-n' touch a.txt ''') wf = script.workflow() Base_Executor(wf).run() self.assertFalse(os.path.exists('a.txt'))
def testPythonsInDocker(self): '''Test action pythons in docker environment''' script = SoS_Script(r''' [0] python3: docker_image='python' #!/usr/bin/env python3 a = {'1', '2'} print(a) ''') wf = script.workflow() Base_Executor(wf).run()
def testZsh(self): '''Test action zsh''' if not shutil.which('zsh'): return script = SoS_Script(r''' [0] zsh: echo "Hello World!", $SHELL ''') wf = script.workflow() Base_Executor(wf).run()
def testPython(self): '''Test python command. This might fail if python3 is the default interpreter''' script = SoS_Script(r''' [0] python: a = {'1': 2} print(a) ''') wf = script.workflow() Base_Executor(wf).run()
def testShInDocker(self): '''Test action sh in docker environment''' # test docker script = SoS_Script(r''' [0] sh: docker_image='ubuntu' echo 'Echo ''') wf = script.workflow() self.assertRaises(ExecuteError, Base_Executor(wf).run) # Base_Executor(wf).dryrun()
def testPerlInDocker(self): '''Test action perl in docker environment''' script = SoS_Script(r''' [0] perl: docker_image='ubuntu' use strict; use warnings; print "hi NAME\n"; ''') wf = script.workflow() Base_Executor(wf).run()
def testR(self): '''Test action JavaScript''' if not shutil.which('R'): return script = SoS_Script(r''' [0] R: nums = rnorm(25, mean=100, sd=15) mean(nums) ''') wf = script.workflow() Base_Executor(wf).run()
def testPerl(self): '''Test action ruby''' script = SoS_Script(r''' [0] perl: use strict; use warnings; print "hi NAME\n"; ''') wf = script.workflow() Base_Executor(wf).run()
def testTcsh(self): '''Test action tcsh''' if not shutil.which('tcsh'): return script = SoS_Script(r''' [0] tcsh: foreach color (red orange yellow green blue) echo $color end ''') wf = script.workflow() Base_Executor(wf).run()
def testMixedTabAndSpace(self): '''Test handling of mixed tab and space''' script = SoS_Script(''' [1: shared=['a', 'b', 'c']] if True: a = 1 \tb = 2 \tc= 3 ''') wf = script.workflow() Base_Executor(wf).run() self.assertEqual(env.sos_dict['a'], 1) self.assertEqual(env.sos_dict['b'], 2) self.assertEqual(env.sos_dict['c'], 3)
def testAnalyzeSection(self): """Test analysis of sections (statically)""" script = SoS_Script( """ g1 = 'a' g2 = 1 parameter: p1 = 5 parameter: infiles = 'a.txt' [A_1: shared='b'] b = p1 + 2 input: infiles output: None c = 5 [A_2] b = [1, 2, 3] input: for_each='b' depends: 'some.txt', executable('ls') import time import random r = random.randint(1, 5) time.sleep(r) [A_3] input: None print(p1) """ ) wf = script.workflow("A") for section in wf.sections: res = analyze_section(section) if section.names[0][1] == "1": self.assertTrue(isinstance(res["step_input"], Undetermined)) self.assertEqual(res["step_depends"], []) self.assertEqual(res["step_output"], []) self.assertEqual(res["environ_vars"], {"p1", "infiles"}) self.assertEqual(res["signature_vars"], {"c"}) self.assertEqual(res["changed_vars"], {"b"}) elif section.names[0][1] == "2": self.assertEqual(res["step_input"], []) self.assertEqual(res["step_depends"], ["some.txt", executable("ls")]) self.assertTrue(isinstance(res["step_output"], Undetermined)) # for_each will not be used for DAG self.assertEqual(res["environ_vars"], {"for_each"}) self.assertEqual(res["signature_vars"], {"import", "r", "time", "random"}) self.assertEqual(res["changed_vars"], set())
def testStopIf(self): '''Test action stop_if''' script = SoS_Script(r''' [0: shared='result'] rep = range(20) result = [] input: for_each='rep' stop_if(_rep > 10) result.append(_rep) ''') wf = script.workflow() Base_Executor(wf).run() self.assertEqual(env.sos_dict['result'], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
def testParallelExecution(self): '''Test basic parallel execution''' ''' A1 <- None A2 <- B2 ''' for f in ['A1.txt', 'B2.txt', 'A2.txt']: FileTarget(f).remove('both') script = SoS_Script(''' [A_1] output: 'A1.txt' sh: sleep 3 touch A1.txt [A_2] input: 'B2.txt' output: 'A2.txt' sh: sleep 3 touch A2.txt [B: provides='B2.txt'] output: 'B2.txt' sh: touch B2.txt ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow() dag = Base_Executor(wf).initialize_dag() self.assertDAG(dag, ''' strict digraph "" { A_1; A_2; "B ['B2.txt']"; "B ['B2.txt']" -> A_2; } ''') env.max_jobs = 4 st = time.time() #env.verbosity = 4 MP_Executor(wf).run() self.assertLess(time.time() - st, 4) for f in ['A1.txt', 'B2.txt', 'A2.txt']: FileTarget(f).remove('both')
def testParallelExecution(self): '''Test basic parallel execution''' ''' A1 <- None A2 <- B2 ''' for f in ['A1.txt', 'B2.txt', 'A2.txt']: FileTarget(f).remove('both') script = SoS_Script(''' [A_1] output: 'A1.txt' sh: sleep 3 touch A1.txt [A_2] input: 'B2.txt' output: 'A2.txt' sh: sleep 3 touch A2.txt [B: provides='B2.txt'] output: 'B2.txt' sh: touch B2.txt ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow() dag = Base_Executor(wf).initialize_dag() self.assertDAG( dag, ''' strict digraph "" { A_1; A_2; "B ['B2.txt']"; "B ['B2.txt']" -> A_2; } ''') env.max_jobs = 4 st = time.time() #env.verbosity = 4 MP_Executor(wf).run() self.assertLess(time.time() - st, 4) for f in ['A1.txt', 'B2.txt', 'A2.txt']: FileTarget(f).remove('both')
def testDynamicNestedWorkflow(self): # # Because we are not sure which workflows would be executed # until run time, the DAG should not contain nested workflow # until runtime. # for f in [ 'B0.txt', 'B0.txt.p', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p' ]: FileTarget(f).remove('both') # # A1 <- P <- B # A1 <- P <- B # A2 # # ALL calls A and B with parameter # script = SoS_Script(''' [A_1] parameter: num = 2 input: "B${num}.txt.p" [B: provides='B{num}.txt'] sh: touch 'B${num}.txt' [P: provides='{filename}.p'] input: filename sh: touch ${output} [ALL] for num in range(3): sos_run('A') ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow('ALL') Base_Executor(wf).run() for f in [ 'B0.txt', 'B0.txt.p', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p' ]: self.assertTrue(FileTarget(f).exists()) FileTarget(f).remove('both')
def testDockerImage(self): '''Test docker_image option''' script = SoS_Script(r''' [0] fastq_files = glob.glob('data/*.fastq') input_volume = os.path.dirname(fastq_files[0]) output_volume = os.getcwd() run: docker_image='compbio/ngseasy-fastqc:1.0-r001', volumes=["${input_volume}:/input_data", "${output_volume}:/output_data"] ls -l /input_data /usr/local/bin/fastqc /input_data/*.fastq --outdir /output_data ''') wf = script.workflow() Base_Executor(wf).run()
def testCell(self): '''Test ignoring %cell''' SoS_Script(''' %cell 1 [step ] a = 1 ''')
def testAnalyzeSection(self): '''Test analysis of sections (statically)''' script = SoS_Script(''' g1 = 'a' g2 = 1 parameter: p1 = 5 parameter: infiles = 'a.txt' [A_1: shared='b'] b = p1 + 2 input: infiles output: None c = 5 [A_2] b = [1, 2, 3] input: for_each='b' depends: 'some.txt', executable('ls') import time import random r = random.randint(1, 5) time.sleep(r) [A_3] input: None print(p1) ''') wf = script.workflow('A') for section in wf.sections: res = analyze_section(section) if section.names[0][1] == '1': self.assertTrue(isinstance(res['step_input'], Undetermined)) self.assertEqual(res['step_depends'], []) self.assertEqual(res['step_output'], []) self.assertEqual(res['environ_vars'], {'p1', 'infiles'}) self.assertEqual(res['signature_vars'], {'c'}) self.assertEqual(res['changed_vars'], {'b'}) elif section.names[0][1] == '2': self.assertEqual(res['step_input'], []) self.assertEqual(res['step_depends'], ['some.txt', executable('ls')]) self.assertTrue(isinstance(res['step_output'], Undetermined)) # for_each will not be used for DAG self.assertEqual(res['environ_vars'], {'for_each'}) self.assertEqual(res['signature_vars'], {'import', 'r', 'time', 'random'}) self.assertEqual(res['changed_vars'], set())
def testRubyInDocker(self): '''Test action ruby in docker environment''' script = SoS_Script(r''' [0] ruby: docker_image='ruby' line1 = "Cats are smarter than dogs"; line2 = "Dogs also like meat"; if ( line1 =~ /Cats(.*)/ ) puts "Line1 contains Cats" end if ( line2 =~ /Cats(.*)/ ) puts "Line2 contains Dogs" end ''') wf = script.workflow() Base_Executor(wf).run()
def testSharedDependency(self): # # shared variable should introduce additional dependency # for f in ['A1.txt']: FileTarget(f).remove('both') # # A1 introduces a shared variable ss, A3 depends on ss but not A2 # script = SoS_Script(''' [A_1: shared='ss'] ss = 'A1' [A_2] input: None sh: sleep 3 [A_3] input: None import time time.sleep(3) with open("${ss}.txt", 'w') as tmp: tmp.write('test') ''') wf = script.workflow('A') dag = Base_Executor(wf).initialize_dag() self.assertDAG( dag, ''' strict digraph "" { A_3; A_1; A_2; A_1 -> A_3; } ''') env.max_jobs = 3 st = time.time() MP_Executor(wf).run() self.assertLess(time.time() - st, 5) for f in ['A1.txt']: self.assertTrue(FileTarget(f).exists()) FileTarget(f).remove('both')
def testSharedDependency(self): # # shared variable should introduce additional dependency # for f in ['A1.txt']: FileTarget(f).remove('both') # # A1 introduces a shared variable ss, A3 depends on ss but not A2 # script = SoS_Script(''' [A_1: shared='ss'] ss = 'A1' [A_2] input: None sh: sleep 3 [A_3] input: None import time time.sleep(3) with open("${ss}.txt", 'w') as tmp: tmp.write('test') ''') wf = script.workflow('A') dag = Base_Executor(wf).initialize_dag() self.assertDAG(dag, ''' strict digraph "" { A_3; A_1; A_2; A_1 -> A_3; } ''') env.max_jobs = 3 st = time.time() MP_Executor(wf).run() self.assertLess(time.time() - st, 5) for f in ['A1.txt']: self.assertTrue(FileTarget(f).exists()) FileTarget(f).remove('both')
def testSkipStep(self): '''Test the skip option to skip certain steps''' script = SoS_Script(''' parameter: skip = 0 [0: shared={'a':'var'}, skip=skip==0] var = 0 [1: shared={'b': 'var'}, skip=skip==1] var = 1 ''') wf = script.workflow() Base_Executor(wf, args=['--skip', '0']).run() self.assertEqual(env.sos_dict['b'], 1) # Base_Executor(wf, args=['--skip', '1']).run() self.assertEqual(env.sos_dict['a'], 0)
def testVariableTarget(self): '''Test dependency caused by variable usage.''' script = SoS_Script(r''' [A: shared='b'] b = 1 [C: shared={'c':'k'}] k = 2 [all: shared='p'] depends: sos_variable('c'), sos_variable('b') p = c + b ''') wf = script.workflow('all') Base_Executor(wf).run() self.assertTrue(env.sos_dict['p'], 3)
def testCombinedWorkflow(self): '''Test the creation and execution of combined workfow''' script = SoS_Script(''' a0 = 0 if 'executed' not in locals(): executed = [] parameter: a = a0 + 1 [a_1: shared='executed'] executed.append(step_name) [a_2: shared='executed'] executed.append(step_name) [a_3: shared='executed'] executed.append(step_name) [a_4: shared='executed'] executed.append(step_name) output: 'out_a_4' [b_1: shared=['executed', 'input_b1']] executed.append(step_name) input_b1 = input [b_2: shared='executed'] executed.append(step_name) [b_3: shared='executed'] executed.append(step_name) [b_4: shared='executed'] executed.append(step_name) [c: shared='executed'] executed.append(step_name) [d: shared='executed'] executed.append(step_name) ''') wf = script.workflow('a+b') Base_Executor(wf).dryrun() self.assertEqual(env.sos_dict['executed'], ['a_1', 'a_2', 'a_3', 'a_4', 'b_1', 'b_2', 'b_3', 'b_4']) self.assertEqual(env.sos_dict['a'], 1) self.assertEqual(env.sos_dict['input_b1'], ['out_a_4']) # wf = script.workflow('a: 1-2 + a:4 + b:3-') Base_Executor(wf).dryrun() self.assertEqual(env.sos_dict['executed'], ['a_1', 'a_2', 'a_4', 'b_3', 'b_4']) # wf = script.workflow('a+c+d') Base_Executor(wf).dryrun() self.assertEqual(env.sos_dict['executed'], ['a_1', 'a_2', 'a_3', 'a_4', 'c_0', 'd_0'])
def testDockerBuild(self): '''Test action docker build''' script = SoS_Script(r''' [0] docker_build: tag='test/docker_build' # # Super simple example of a Dockerfile # FROM ubuntu:latest MAINTAINER Andrew Odewahn "*****@*****.**" RUN apt-get update RUN apt-get install -y python python-pip wget RUN pip install Flask WORKDIR /home ''') wf = script.workflow() Base_Executor(wf).run()
def testDockerImageFromFile(self): '''Test docker_image load from a file.''' # image from a saved file script = SoS_Script(r''' [0] run: docker_image='blang/busybox-bash' [1] run: docker save blang/busybox-bash > hello.tar docker rmi -f blang/busybox-bash [2] run: docker_image='blang/busybox-bash', docker_file = 'hello.tar' echo "a" ''') wf = script.workflow() Base_Executor(wf).run()
def testProgressBar(self): """Test progress bar""" env.verbosity = 1 prog = ProgressBar("test", 100) for i in range(100): prog.update(i) prog.done() prog = ProgressBar("test", 100) for i in range(20): prog.progress(5) prog.done() # script = SoS_Script( """ [1] [2] [3] [4] [5] """ ) wf = script.workflow() Base_Executor(wf).run() # progress bar with nested workflow script = SoS_Script( """ import time time.sleep(0.5) [sub_1] [sub_2] [sub_3] [sub_4] [a_1] [a_2] [a_3] sos_run('sub') [a_4] [a_5] """ ) wf = script.workflow("a") Base_Executor(wf).run()