def testInput(self): '''Test input directive''' self.touch(['a.txt', 'b.txt', 'a.pdf', 'a0', 'a1']) script = SoS_Script(''' [0] files = ['a.txt', 'b.txt'] input: 'a.pdf', files ''') wf = script.workflow('default') Base_Executor(wf).dryrun() # # test input types script = SoS_Script(''' [0:shared={'i':'input', 'o':'output'}] files = ("a${i}" for i in range(2)) input: {'a.txt', 'b.txt'}, files output: ("a${x}" for x in _input) ''') wf = script.workflow() Base_Executor(wf).dryrun() self.assertEqual(sorted(env.sos_dict['i']), ['a.txt', 'a0', 'a1', 'b.txt']) self.assertEqual(sorted(env.sos_dict['o']), ['aa.txt', 'aa0', 'aa1', 'ab.txt'])
def testSoSAction(self): '''Test sos_action decorator''' script = SoS_Script(r""" from sos.actions import SoS_Action @SoS_Action(run_mode='run') def func_run(): return 1 @SoS_Action(run_mode=['run', 'dryrun']) def func_both(): return 1 [0: shared=('b', 'c')] b=func_run() c=func_both() """) wf = script.workflow() Base_Executor(wf).dryrun() self.assertTrue(isinstance(env.sos_dict['b'], Undetermined)) self.assertTrue(isinstance(env.sos_dict['c'], Undetermined)) # Base_Executor(wf).run() self.assertEqual(env.sos_dict['b'], 1) self.assertEqual(env.sos_dict['c'], 1)
def testFromInclude(self): '''Test include keyword''' with open('inc.sos', 'w') as ts: ts.write(''' # a slave script to be included gv = 1 [A_1] [A_2] [B] ''') script = SoS_Script(''' %from inc include gv [0] ''') wf = script.workflow() Base_Executor(wf).dryrun() Base_Executor(wf).run() self.assertEqual(env.sos_dict['gv'], 1) # # include with alias script = SoS_Script(''' %from inc include gv as g res1 = g [0] ''') wf = script.workflow() Base_Executor(wf).dryrun() Base_Executor(wf).run() self.assertEqual(env.sos_dict['res1'], 1)
def testOverwriteKeyword(self): '''Test overwrite sos keyword with user defined one.''' FileTarget('a.txt').remove('both') # script = SoS_Script(''' def run(script): pass [1] run: touch a.txt ''') wf = script.workflow() Base_Executor(wf).run() self.assertFalse(os.path.isfile('a.txt')) # script = SoS_Script(''' parameter: run = 5 [1] run: touch a.txt ''') wf = script.workflow() self.assertRaises(Exception, Base_Executor(wf).run)
def testVarOutput(self): '''Test early appearance of variable output''' script = SoS_Script(''' [0] seq = range(3) input: for_each='seq' output: "test${_seq}.txt" print(output) ''') wf = script.workflow() # this does not work before until we make variable output available sooner Base_Executor(wf).dryrun() # however, output should be the same in task script = SoS_Script(''' [0] seq = range(3) input: for_each='seq' output: "test${_seq}.txt" assert(len(output), _index + 1) task: assert(len(output), 3) ''') wf = script.workflow() # this does not work before until we make variable output available sooner Base_Executor(wf).dryrun()
def testInclude(self): '''Test include keyword''' with open('inc.sos', 'w') as ts: ts.write(''' # a slave script to be included gv = 1 [A_1] [A_2] [B] ''') script = SoS_Script(''' %include inc res = inc.gv [0] ''') wf = script.workflow() Base_Executor(wf).dryrun() Base_Executor(wf).run() self.assertEqual(env.sos_dict['res'], 1) # # include with alias script = SoS_Script(''' %include inc as tt res1 = tt.gv [0] ''') wf = script.workflow() Base_Executor(wf).dryrun() Base_Executor(wf).run() self.assertEqual(env.sos_dict['res1'], 1) os.remove('inc.sos')
def testDynamicNestedWorkflow(self): '''Test nested workflow controlled by command line option''' script = SoS_Script(''' if 'executed' not in locals(): executed = [] parameter: wf='a' [a_1:shared='executed'] executed.append(step_name) [a_2:shared='executed'] executed.append(step_name) [a_3:shared='executed'] executed.append(step_name) [b_1:shared='executed'] executed.append(step_name) [b_2:shared='executed'] executed.append(step_name) [b_3:shared='executed'] executed.append(step_name) [default:shared='executed'] executed.append(step_name) sos_run(wf) ''') wf = script.workflow() Base_Executor(wf, args=['--wf', 'b']).run() self.assertEqual(env.sos_dict['executed'], [ 'default_0', 'b_1', 'b_2', 'b_3']) # Base_Executor(wf, args=['--wf', 'a']).run() self.assertEqual(env.sos_dict['executed'], ['default_0', 'a_1', 'a_2', 'a_3'])
def testShInDocker(self): '''Test action sh in docker environment''' # test docker script = SoS_Script(r''' [0] sh: docker_image='ubuntu' echo 'Echo ''') wf = script.workflow() self.assertRaises(ExecuteError, Base_Executor(wf).run) # Base_Executor(wf).dryrun()
def testSh(self): '''Test action run''' script = SoS_Script(r''' [0] sh: echo 'Echo' ''') wf = script.workflow() Base_Executor(wf).run() script = SoS_Script(r''' [0] sh: echo 'Echo ''') wf = script.workflow() self.assertRaises(ExecuteError, Base_Executor(wf).run)
def testChainedDepends(self): '''Test chain dependent''' shutil.rmtree('.sos') os.makedirs('.sos/.runtime') script = SoS_Script(r''' # this step provides variable `var` [index: provides='{filename}.bam.bai'] input: "${filename}.bam" sh: echo "Generating ${output}" touch ${output} [call: provides='{filename}.vcf'] input: "${filename}.bam" depends: "${input}.bai" sh: echo "Calling variants from ${input} with ${depends} to ${output}" touch ${output} ''') FileTarget('a.bam.bai').remove('both') FileTarget('a.vcf').remove('both') self.touch('a.bam') Base_Executor(script.workflow()).run(targets=['a.vcf']) for file in ('a.vcf', 'a.bam', 'a.bam.bai'): FileTarget(file).remove('both')
def testGetOutput(self): '''Test utility function get_output''' script = SoS_Script(r""" [0: shared='ret'] ret = get_output('echo blah') """) wf = script.workflow() # should be ok Base_Executor(wf).run() self.assertEqual(env.sos_dict['ret'], 'blah\n') # script = SoS_Script(r""" [0: shared='ret'] ret = get_output('echo blah', show_command=True) """) wf = script.workflow() # should be ok Base_Executor(wf).run() self.assertEqual(env.sos_dict['ret'], '$ echo blah\nblah\n') # script = SoS_Script(r""" [0: shared='ret'] ret = get_output('echo blah', show_command=True, prompt='% ') """) wf = script.workflow() # should be ok Base_Executor(wf).run() self.assertEqual(env.sos_dict['ret'], '% echo blah\nblah\n') # script = SoS_Script(r""" [0] get_output('catmouse') """) wf = script.workflow() # should fail in dryrun mode self.assertRaises(ExecuteError, Base_Executor(wf).run) # # script = SoS_Script(r""" [0] ret = get_output('cat -h') """) wf = script.workflow() # this should give a warning and return false self.assertRaises(ExecuteError, Base_Executor(wf).run)
def testBashInDocker(self): '''Test action bash in docker environment''' script = SoS_Script(r''' [0] bash: docker_image='ubuntu' echo 'Echo' ''') wf = script.workflow() Base_Executor(wf).run()
def testLiteralConnection(self): '''Testing the connection of steps with by variables.''' for f in ['A1.txt']: FileTarget(f).remove('both') # # A1 introduces a shared variable ss, A3 depends on ss but not A2 # script = SoS_Script(''' [A_1: shared='p'] sh: touch 'A1.txt' p = 'A1.txt' [A_2] input: None sh: sleep 3 [A_3] input: p sh: sleep 3 [A_4] input: p sh: sleep 3 [A_5] input: dynamic(p) ''') wf = script.workflow('A') dag = Base_Executor(wf).initialize_dag() self.assertDAG( dag, ''' strict digraph "" { A_1; A_4; A_2; A_3; A_5; A_1 -> A_4; A_1 -> A_3; A_1 -> A_5; A_4 -> A_5; } ''') env.max_jobs = 3 st = time.time() MP_Executor(wf).run() self.assertLess(time.time() - st, 5) for f in ['A1.txt']: self.assertTrue(FileTarget(f).exists()) FileTarget(f).remove('both')
def testSkipStep(self): '''Test the skip option to skip certain steps''' script = SoS_Script(''' parameter: skip = 0 [0: shared={'a':'var'}, skip=skip==0] var = 0 [1: shared={'b': 'var'}, skip=skip==1] var = 1 ''') wf = script.workflow() Base_Executor(wf, args=['--skip', '0']).run() self.assertEqual(env.sos_dict['b'], 1) # Base_Executor(wf, args=['--skip', '1']).run() self.assertEqual(env.sos_dict['a'], 0)
def testYAMLConfig(self): '''Test config file in yaml format''' with open('myconfig.yaml', 'w') as config: config.write(''' # Lines beginning with # are skipped when the JSON is parsed, so we can # put comments into our JSON configuration files { StoreOwner : "John Doe", # List of items that we sell Fruits: [ "apple", "banana", "pear" ], Price: 1.05 } ''') with open('config.sos', 'w') as sos: sos.write(''' [0] print(CONFIG['StoreOwner']) print(CONFIG.get('StoreOwner', 'something')) print(CONFIG.get('StoreOwnerSpouse', 'someone else')) print(CONFIG.StoreOwner) ''' ) # run the command self.assertEqual(subprocess.call('sos run config.sos -c myconfig.yaml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 0) # now test the value script = SoS_Script(filename='config.sos') wf = script.workflow() Base_Executor(wf, config_file='myconfig.yaml').run() self.assertEqual(env.sos_dict['CONFIG']['Price'], 1.05) self.assertEqual(env.sos_dict['CONFIG']['StoreOwner'], 'John Doe') self.assertEqual(env.sos_dict['CONFIG']['Fruits'], ['apple', 'banana', 'pear']) # configuration items should be readonly with open('config.sos', 'w') as sos: sos.write(''' [0] CONFIG['a'] = 'b' ''' ) # the command would fail with error message # ERROR: Failed to process statement CONFIG['a'] = 'b' # : Cannot modify a readonly dictionary. self.assertEqual(subprocess.call('sos run config.sos -c myconfig.yaml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 1) # with open('config.sos', 'w') as sos: sos.write(''' [0] CONFIG.a = 'b' ''' ) # the command would fail with error message # ERROR: Failed to process statement CONFIG['a'] = 'b' # : Cannot modify a readonly dictionary. self.assertEqual(subprocess.call('sos run config.sos -c myconfig.yaml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 1) # for filename in ['config.sos', 'myconfig.yaml']: os.remove(filename)
def testPython3(self): script = SoS_Script(r''' [0] python3: a = {'1', '2'} print(a) ''') wf = script.workflow() Base_Executor(wf).run()
def testCombinedWorkflow(self): '''Test the creation and execution of combined workfow''' script = SoS_Script(''' a0 = 0 if 'executed' not in locals(): executed = [] parameter: a = a0 + 1 [a_1: shared='executed'] executed.append(step_name) [a_2: shared='executed'] executed.append(step_name) [a_3: shared='executed'] executed.append(step_name) [a_4: shared='executed'] executed.append(step_name) output: 'out_a_4' [b_1: shared=['executed', 'input_b1']] executed.append(step_name) input_b1 = input [b_2: shared='executed'] executed.append(step_name) [b_3: shared='executed'] executed.append(step_name) [b_4: shared='executed'] executed.append(step_name) [c: shared='executed'] executed.append(step_name) [d: shared='executed'] executed.append(step_name) ''') wf = script.workflow('a+b') Base_Executor(wf).dryrun() self.assertEqual(env.sos_dict['executed'], ['a_1', 'a_2', 'a_3', 'a_4', 'b_1', 'b_2', 'b_3', 'b_4']) self.assertEqual(env.sos_dict['a'], 1) self.assertEqual(env.sos_dict['input_b1'], ['out_a_4']) # wf = script.workflow('a: 1-2 + a:4 + b:3-') Base_Executor(wf).dryrun() self.assertEqual(env.sos_dict['executed'], ['a_1', 'a_2', 'a_4', 'b_3', 'b_4']) # wf = script.workflow('a+c+d') Base_Executor(wf).dryrun() self.assertEqual(env.sos_dict['executed'], ['a_1', 'a_2', 'a_3', 'a_4', 'c_0', 'd_0'])
def testRInDocker(self): '''Test action R in docker environment''' script = SoS_Script(r''' [0] R: docker_image='r-base' nums = rnorm(25, mean=100, sd=15) mean(nums) ''') wf = script.workflow() Base_Executor(wf).run()
def testPythonInDocker(self): '''Test action python in docker environment''' script = SoS_Script(r''' [0] python: docker_image='python' a = {'1': 2} print(a) ''') wf = script.workflow() Base_Executor(wf).run()
def testPython(self): '''Test python command. This might fail if python3 is the default interpreter''' script = SoS_Script(r''' [0] python: a = {'1': 2} print(a) ''') wf = script.workflow() Base_Executor(wf).run()
def testZsh(self): '''Test action zsh''' if not shutil.which('zsh'): return script = SoS_Script(r''' [0] zsh: echo "Hello World!", $SHELL ''') wf = script.workflow() Base_Executor(wf).run()
def testArgs(self): '''Test args option of scripts''' FileTarget('a.txt').remove('both') script = SoS_Script(r''' [0] sh: args='-n' touch a.txt ''') wf = script.workflow() Base_Executor(wf).run() self.assertFalse(os.path.exists('a.txt'))
def testPythonsInDocker(self): '''Test action pythons in docker environment''' script = SoS_Script(r''' [0] python3: docker_image='python' #!/usr/bin/env python3 a = {'1', '2'} print(a) ''') wf = script.workflow() Base_Executor(wf).run()
def testNodeInDocker(self): '''Test action node in docker environment''' script = SoS_Script(r''' [0] node: docker_image='node' var args = process.argv.slice(2); console.log('Hello ' + args.join(' ') + '!'); ''') wf = script.workflow() Base_Executor(wf).run() # script = SoS_Script(r''' [0] JavaScript: docker_image='node' var args = process.argv.slice(2); console.log('Hello ' + args.join(' ') + '!'); ''') wf = script.workflow() Base_Executor(wf).run()
def testPerlInDocker(self): '''Test action perl in docker environment''' script = SoS_Script(r''' [0] perl: docker_image='ubuntu' use strict; use warnings; print "hi NAME\n"; ''') wf = script.workflow() Base_Executor(wf).run()
def testPerl(self): '''Test action ruby''' script = SoS_Script(r''' [0] perl: use strict; use warnings; print "hi NAME\n"; ''') wf = script.workflow() Base_Executor(wf).run()
def testR(self): '''Test action JavaScript''' if not shutil.which('R'): return script = SoS_Script(r''' [0] R: nums = rnorm(25, mean=100, sd=15) mean(nums) ''') wf = script.workflow() Base_Executor(wf).run()
def testNode(self): '''Test action ruby''' if not shutil.which('node'): return script = SoS_Script(r''' [0] node: var args = process.argv.slice(2); console.log('Hello ' + args.join(' ') + '!'); ''') wf = script.workflow() Base_Executor(wf).run() # script = SoS_Script(r''' [0] JavaScript: var args = process.argv.slice(2); console.log('Hello ' + args.join(' ') + '!'); ''') wf = script.workflow() Base_Executor(wf).run()
def testProgressBar(self): '''Test progress bar''' env.verbosity = 1 prog = ProgressBar('test', 100) for i in range(100): prog.update(i) prog.done() prog = ProgressBar('test', 100) for i in range(20): prog.progress(5) prog.done() # script = SoS_Script(''' [1] [2] [3] [4] [5] ''') wf = script.workflow() Base_Executor(wf).run() # progress bar with nested workflow script = SoS_Script(''' import time time.sleep(0.5) [sub_1] [sub_2] [sub_3] [sub_4] [a_1] [a_2] [a_3] sos_run('sub') [a_4] [a_5] ''') wf = script.workflow('a') Base_Executor(wf).run()
def __init__(self, workflow=None, args=[], nested=False, config={}): # we actually do not have our own workflow, everything is passed from ipython # by nested = True we actually mean no new dictionary if env.sig_mode is None: env.sig_mode = 'ignore' Base_Executor.__init__(self, workflow=workflow, args=args, nested=True, config=config) env.__task_engine__ = 'interactive' if env.sig_mode != 'ignore': self.md5 = self.create_signature() # We append to existing workflow files because some files are ignored and we # still wants their information. with open(os.path.join(env.exec_dir, '.sos', '{}.sig'.format(self.md5)), 'a') as sig: sig.write('# workflow: {}\n'.format(self.workflow.name)) # script is None because it is entered from notebook with open('__interactive__.sos', 'w') as script: script.write(self.workflow.content.content) sig.write('# script: {}\n'.format('__interactive__.sos')) sig.write('# included: {}\n'.format(','.join(self.workflow.content.included))) sig.write('# configuration: {}\n'.format(config.get('config_file', ''))) sig.write('# start time: {}\n'.format(time.strftime('%a, %d %b %Y %H:%M:%S +0000', time.gmtime()))) sig.write(self.sig_content) sig.write('# runtime signatures\n')
def testTcsh(self): '''Test action tcsh''' if not shutil.which('tcsh'): return script = SoS_Script(r''' [0] tcsh: foreach color (red orange yellow green blue) echo $color end ''') wf = script.workflow() Base_Executor(wf).run()
def testUndetermined(self): '''Test DAG with undetermined input.''' # for filename in ('a.txt', 'd.txt'): with open(filename, 'w') as tmp: tmp.write('hey') # input of step 3 is undertermined so # it depends on all its previous steps. script = SoS_Script(''' [C_1] input: 'a.txt' output: 'b.txt' [C_2] input: 'b.txt' output: 'c.txt' [C_3] input: dynamic('*.txt') output: 'd.txt' [C_4] depends: 'd.txt' output: 'e.txt' ''') wf = script.workflow() dag = Base_Executor(wf).initialize_dag() dag.show_nodes() #dag.write_dot('a.dot') self.assertDAG(dag, ''' strict digraph "" { C_1; C_4; C_2; C_3; C_1 -> C_2; C_2 -> C_3; C_3 -> C_4; } ''') # # output of step # script = SoS_Script(''' [C_1] input: 'a.txt' output: 'b.txt' [C_2] input: 'b.txt' output: 'c.txt' [C_3] input: dynamic('*.txt') [C_4] depends: 'd.txt' output: 'e.txt' ''') wf = script.workflow() dag = Base_Executor(wf).initialize_dag() self.assertDAG(dag, ''' strict digraph "" { C_1; C_4; C_2; C_3; C_1 -> C_2; C_2 -> C_3; C_3 -> C_4; } ''') for filename in ('a.txt', 'd.txt'): os.remove(filename)