def testInput(self):
'''Test input directive'''
self.touch(['a.txt', 'b.txt', 'a.pdf', 'a0', 'a1'])
script = SoS_Script('''
[0]
files = ['a.txt', 'b.txt']
input: 'a.pdf', files
''')
wf = script.workflow('default')
Base_Executor(wf).dryrun()
#
# test input types
script = SoS_Script('''
[0:shared={'i':'input', 'o':'output'}]
files = ("a${i}" for i in range(2))
input: {'a.txt', 'b.txt'}, files
output: ("a${x}" for x in _input)
''')
wf = script.workflow()
Base_Executor(wf).dryrun()
self.assertEqual(sorted(env.sos_dict['i']), ['a.txt', 'a0', 'a1', 'b.txt'])
self.assertEqual(sorted(env.sos_dict['o']), ['aa.txt', 'aa0', 'aa1', 'ab.txt'])
def testSoSAction(self):
'''Test sos_action decorator'''
script = SoS_Script(r"""
from sos.actions import SoS_Action
@SoS_Action(run_mode='run')
def func_run():
return 1
@SoS_Action(run_mode=['run', 'dryrun'])
def func_both():
return 1
[0: shared=('b', 'c')]
b=func_run()
c=func_both()
""")
wf = script.workflow()
Base_Executor(wf).dryrun()
self.assertTrue(isinstance(env.sos_dict['b'], Undetermined))
self.assertTrue(isinstance(env.sos_dict['c'], Undetermined))
#
Base_Executor(wf).run()
self.assertEqual(env.sos_dict['b'], 1)
self.assertEqual(env.sos_dict['c'], 1)
def testFromInclude(self):
'''Test include keyword'''
with open('inc.sos', 'w') as ts:
ts.write('''
# a slave script to be included
gv = 1
[A_1]
[A_2]
[B]
''')
script = SoS_Script('''
%from inc include gv
[0]
''')
wf = script.workflow()
Base_Executor(wf).dryrun()
Base_Executor(wf).run()
self.assertEqual(env.sos_dict['gv'], 1)
#
# include with alias
script = SoS_Script('''
%from inc include gv as g
res1 = g
[0]
''')
wf = script.workflow()
Base_Executor(wf).dryrun()
Base_Executor(wf).run()
self.assertEqual(env.sos_dict['res1'], 1)
def testOverwriteKeyword(self):
'''Test overwrite sos keyword with user defined one.'''
FileTarget('a.txt').remove('both')
#
script = SoS_Script('''
def run(script):
pass
[1]
run:
touch a.txt
''')
wf = script.workflow()
Base_Executor(wf).run()
self.assertFalse(os.path.isfile('a.txt'))
#
script = SoS_Script('''
parameter: run = 5
[1]
run:
touch a.txt
''')
wf = script.workflow()
self.assertRaises(Exception, Base_Executor(wf).run)
def testVarOutput(self):
'''Test early appearance of variable output'''
script = SoS_Script('''
[0]
seq = range(3)
input: for_each='seq'
output: "test${_seq}.txt"
print(output)
''')
wf = script.workflow()
# this does not work before until we make variable output available sooner
Base_Executor(wf).dryrun()
# however, output should be the same in task
script = SoS_Script('''
[0]
seq = range(3)
input: for_each='seq'
output: "test${_seq}.txt"
assert(len(output), _index + 1)
task:
assert(len(output), 3)
''')
wf = script.workflow()
# this does not work before until we make variable output available sooner
Base_Executor(wf).dryrun()
def testInclude(self):
'''Test include keyword'''
with open('inc.sos', 'w') as ts:
ts.write('''
# a slave script to be included
gv = 1
[A_1]
[A_2]
[B]
''')
script = SoS_Script('''
%include inc
res = inc.gv
[0]
''')
wf = script.workflow()
Base_Executor(wf).dryrun()
Base_Executor(wf).run()
self.assertEqual(env.sos_dict['res'], 1)
#
# include with alias
script = SoS_Script('''
%include inc as tt
res1 = tt.gv
[0]
''')
wf = script.workflow()
Base_Executor(wf).dryrun()
Base_Executor(wf).run()
self.assertEqual(env.sos_dict['res1'], 1)
os.remove('inc.sos')
def testDynamicNestedWorkflow(self):
'''Test nested workflow controlled by command line option'''
script = SoS_Script('''
if 'executed' not in locals():
executed = []
parameter: wf='a'
[a_1:shared='executed']
executed.append(step_name)
[a_2:shared='executed']
executed.append(step_name)
[a_3:shared='executed']
executed.append(step_name)
[b_1:shared='executed']
executed.append(step_name)
[b_2:shared='executed']
executed.append(step_name)
[b_3:shared='executed']
executed.append(step_name)
[default:shared='executed']
executed.append(step_name)
sos_run(wf)
''')
wf = script.workflow()
Base_Executor(wf, args=['--wf', 'b']).run()
self.assertEqual(env.sos_dict['executed'], [ 'default_0', 'b_1', 'b_2', 'b_3'])
#
Base_Executor(wf, args=['--wf', 'a']).run()
self.assertEqual(env.sos_dict['executed'], ['default_0', 'a_1', 'a_2', 'a_3'])
def testShInDocker(self):
'''Test action sh in docker environment'''
# test docker
script = SoS_Script(r'''
[0]
sh: docker_image='ubuntu'
echo 'Echo
''')
wf = script.workflow()
self.assertRaises(ExecuteError, Base_Executor(wf).run)
#
Base_Executor(wf).dryrun()
def testSh(self):
'''Test action run'''
script = SoS_Script(r'''
[0]
sh:
echo 'Echo'
''')
wf = script.workflow()
Base_Executor(wf).run()
script = SoS_Script(r'''
[0]
sh:
echo 'Echo
''')
wf = script.workflow()
self.assertRaises(ExecuteError, Base_Executor(wf).run)
def testChainedDepends(self):
'''Test chain dependent'''
shutil.rmtree('.sos')
os.makedirs('.sos/.runtime')
script = SoS_Script(r'''
# this step provides variable `var`
[index: provides='{filename}.bam.bai']
input: "${filename}.bam"
sh:
echo "Generating ${output}"
touch ${output}
[call: provides='{filename}.vcf']
input: "${filename}.bam"
depends: "${input}.bai"
sh:
echo "Calling variants from ${input} with ${depends} to ${output}"
touch ${output}
''')
FileTarget('a.bam.bai').remove('both')
FileTarget('a.vcf').remove('both')
self.touch('a.bam')
Base_Executor(script.workflow()).run(targets=['a.vcf'])
for file in ('a.vcf', 'a.bam', 'a.bam.bai'):
FileTarget(file).remove('both')
def testGetOutput(self):
'''Test utility function get_output'''
script = SoS_Script(r"""
[0: shared='ret']
ret = get_output('echo blah')
""")
wf = script.workflow()
# should be ok
Base_Executor(wf).run()
self.assertEqual(env.sos_dict['ret'], 'blah\n')
#
script = SoS_Script(r"""
[0: shared='ret']
ret = get_output('echo blah', show_command=True)
""")
wf = script.workflow()
# should be ok
Base_Executor(wf).run()
self.assertEqual(env.sos_dict['ret'], '$ echo blah\nblah\n')
#
script = SoS_Script(r"""
[0: shared='ret']
ret = get_output('echo blah', show_command=True, prompt='% ')
""")
wf = script.workflow()
# should be ok
Base_Executor(wf).run()
self.assertEqual(env.sos_dict['ret'], '% echo blah\nblah\n')
#
script = SoS_Script(r"""
[0]
get_output('catmouse')
""")
wf = script.workflow()
# should fail in dryrun mode
self.assertRaises(ExecuteError, Base_Executor(wf).run)
#
#
script = SoS_Script(r"""
[0]
ret = get_output('cat -h')
""")
wf = script.workflow()
# this should give a warning and return false
self.assertRaises(ExecuteError, Base_Executor(wf).run)
def testBashInDocker(self):
'''Test action bash in docker environment'''
script = SoS_Script(r'''
[0]
bash: docker_image='ubuntu'
echo 'Echo'
''')
wf = script.workflow()
Base_Executor(wf).run()
def testLiteralConnection(self):
'''Testing the connection of steps with by variables.'''
for f in ['A1.txt']:
FileTarget(f).remove('both')
#
# A1 introduces a shared variable ss, A3 depends on ss but not A2
#
script = SoS_Script('''
[A_1: shared='p']
sh:
touch 'A1.txt'
p = 'A1.txt'
[A_2]
input: None
sh:
sleep 3
[A_3]
input: p
sh:
sleep 3
[A_4]
input: p
sh:
sleep 3
[A_5]
input: dynamic(p)
''')
wf = script.workflow('A')
dag = Base_Executor(wf).initialize_dag()
self.assertDAG(
dag, '''
strict digraph "" {
A_1;
A_4;
A_2;
A_3;
A_5;
A_1 -> A_4;
A_1 -> A_3;
A_1 -> A_5;
A_4 -> A_5;
}
''')
env.max_jobs = 3
st = time.time()
MP_Executor(wf).run()
self.assertLess(time.time() - st, 5)
for f in ['A1.txt']:
self.assertTrue(FileTarget(f).exists())
FileTarget(f).remove('both')
def testSkipStep(self):
'''Test the skip option to skip certain steps'''
script = SoS_Script('''
parameter: skip = 0
[0: shared={'a':'var'}, skip=skip==0]
var = 0
[1: shared={'b': 'var'}, skip=skip==1]
var = 1
''')
wf = script.workflow()
Base_Executor(wf, args=['--skip', '0']).run()
self.assertEqual(env.sos_dict['b'], 1)
#
Base_Executor(wf, args=['--skip', '1']).run()
self.assertEqual(env.sos_dict['a'], 0)
def testYAMLConfig(self):
'''Test config file in yaml format'''
with open('myconfig.yaml', 'w') as config:
config.write('''
# Lines beginning with # are skipped when the JSON is parsed, so we can
# put comments into our JSON configuration files
{
StoreOwner : "John Doe",
# List of items that we sell
Fruits: [ "apple", "banana", "pear" ],
Price: 1.05
}
''')
with open('config.sos', 'w') as sos:
sos.write('''
[0]
print(CONFIG['StoreOwner'])
print(CONFIG.get('StoreOwner', 'something'))
print(CONFIG.get('StoreOwnerSpouse', 'someone else'))
print(CONFIG.StoreOwner)
'''
)
# run the command
self.assertEqual(subprocess.call('sos run config.sos -c myconfig.yaml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 0)
# now test the value
script = SoS_Script(filename='config.sos')
wf = script.workflow()
Base_Executor(wf, config_file='myconfig.yaml').run()
self.assertEqual(env.sos_dict['CONFIG']['Price'], 1.05)
self.assertEqual(env.sos_dict['CONFIG']['StoreOwner'], 'John Doe')
self.assertEqual(env.sos_dict['CONFIG']['Fruits'], ['apple', 'banana', 'pear'])
# configuration items should be readonly
with open('config.sos', 'w') as sos:
sos.write('''
[0]
CONFIG['a'] = 'b'
'''
)
# the command would fail with error message
# ERROR: Failed to process statement CONFIG['a'] = 'b'
# : Cannot modify a readonly dictionary.
self.assertEqual(subprocess.call('sos run config.sos -c myconfig.yaml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 1)
#
with open('config.sos', 'w') as sos:
sos.write('''
[0]
CONFIG.a = 'b'
'''
)
# the command would fail with error message
# ERROR: Failed to process statement CONFIG['a'] = 'b'
# : Cannot modify a readonly dictionary.
self.assertEqual(subprocess.call('sos run config.sos -c myconfig.yaml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 1)
#
for filename in ['config.sos', 'myconfig.yaml']:
os.remove(filename)
def testPython3(self):
script = SoS_Script(r'''
[0]
python3:
a = {'1', '2'}
print(a)
''')
wf = script.workflow()
Base_Executor(wf).run()
def testCombinedWorkflow(self):
'''Test the creation and execution of combined workfow'''
script = SoS_Script('''
a0 = 0
if 'executed' not in locals():
executed = []
parameter: a = a0 + 1
[a_1: shared='executed']
executed.append(step_name)
[a_2: shared='executed']
executed.append(step_name)
[a_3: shared='executed']
executed.append(step_name)
[a_4: shared='executed']
executed.append(step_name)
output: 'out_a_4'
[b_1: shared=['executed', 'input_b1']]
executed.append(step_name)
input_b1 = input
[b_2: shared='executed']
executed.append(step_name)
[b_3: shared='executed']
executed.append(step_name)
[b_4: shared='executed']
executed.append(step_name)
[c: shared='executed']
executed.append(step_name)
[d: shared='executed']
executed.append(step_name)
''')
wf = script.workflow('a+b')
Base_Executor(wf).dryrun()
self.assertEqual(env.sos_dict['executed'], ['a_1', 'a_2', 'a_3', 'a_4', 'b_1', 'b_2', 'b_3', 'b_4'])
self.assertEqual(env.sos_dict['a'], 1)
self.assertEqual(env.sos_dict['input_b1'], ['out_a_4'])
#
wf = script.workflow('a: 1-2 + a:4 + b:3-')
Base_Executor(wf).dryrun()
self.assertEqual(env.sos_dict['executed'], ['a_1', 'a_2', 'a_4',
'b_3', 'b_4'])
#
wf = script.workflow('a+c+d')
Base_Executor(wf).dryrun()
self.assertEqual(env.sos_dict['executed'], ['a_1', 'a_2', 'a_3', 'a_4', 'c_0', 'd_0'])
def testRInDocker(self):
'''Test action R in docker environment'''
script = SoS_Script(r'''
[0]
R: docker_image='r-base'
nums = rnorm(25, mean=100, sd=15)
mean(nums)
''')
wf = script.workflow()
Base_Executor(wf).run()
def testPythonInDocker(self):
'''Test action python in docker environment'''
script = SoS_Script(r'''
[0]
python: docker_image='python'
a = {'1': 2}
print(a)
''')
wf = script.workflow()
Base_Executor(wf).run()
def testPython(self):
'''Test python command. This might fail if python3 is the
default interpreter'''
script = SoS_Script(r'''
[0]
python:
a = {'1': 2}
print(a)
''')
wf = script.workflow()
Base_Executor(wf).run()
def testZsh(self):
'''Test action zsh'''
if not shutil.which('zsh'):
return
script = SoS_Script(r'''
[0]
zsh:
echo "Hello World!", $SHELL
''')
wf = script.workflow()
Base_Executor(wf).run()
def testArgs(self):
'''Test args option of scripts'''
FileTarget('a.txt').remove('both')
script = SoS_Script(r'''
[0]
sh: args='-n'
touch a.txt
''')
wf = script.workflow()
Base_Executor(wf).run()
self.assertFalse(os.path.exists('a.txt'))
def testPythonsInDocker(self):
'''Test action pythons in docker environment'''
script = SoS_Script(r'''
[0]
python3: docker_image='python'
#!/usr/bin/env python3
a = {'1', '2'}
print(a)
''')
wf = script.workflow()
Base_Executor(wf).run()
def testNodeInDocker(self):
'''Test action node in docker environment'''
script = SoS_Script(r'''
[0]
node: docker_image='node'
var args = process.argv.slice(2);
console.log('Hello ' + args.join(' ') + '!');
''')
wf = script.workflow()
Base_Executor(wf).run()
#
script = SoS_Script(r'''
[0]
JavaScript: docker_image='node'
var args = process.argv.slice(2);
console.log('Hello ' + args.join(' ') + '!');
''')
wf = script.workflow()
Base_Executor(wf).run()
def testPerlInDocker(self):
'''Test action perl in docker environment'''
script = SoS_Script(r'''
[0]
perl: docker_image='ubuntu'
use strict;
use warnings;
print "hi NAME\n";
''')
wf = script.workflow()
Base_Executor(wf).run()
def testPerl(self):
'''Test action ruby'''
script = SoS_Script(r'''
[0]
perl:
use strict;
use warnings;
print "hi NAME\n";
''')
wf = script.workflow()
Base_Executor(wf).run()
def testR(self):
'''Test action JavaScript'''
if not shutil.which('R'):
return
script = SoS_Script(r'''
[0]
R:
nums = rnorm(25, mean=100, sd=15)
mean(nums)
''')
wf = script.workflow()
Base_Executor(wf).run()
def testNode(self):
'''Test action ruby'''
if not shutil.which('node'):
return
script = SoS_Script(r'''
[0]
node:
var args = process.argv.slice(2);
console.log('Hello ' + args.join(' ') + '!');
''')
wf = script.workflow()
Base_Executor(wf).run()
#
script = SoS_Script(r'''
[0]
JavaScript:
var args = process.argv.slice(2);
console.log('Hello ' + args.join(' ') + '!');
''')
wf = script.workflow()
Base_Executor(wf).run()
def testProgressBar(self):
'''Test progress bar'''
env.verbosity = 1
prog = ProgressBar('test', 100)
for i in range(100):
prog.update(i)
prog.done()
prog = ProgressBar('test', 100)
for i in range(20):
prog.progress(5)
prog.done()
#
script = SoS_Script('''
[1]
[2]
[3]
[4]
[5]
''')
wf = script.workflow()
Base_Executor(wf).run()
# progress bar with nested workflow
script = SoS_Script('''
import time
time.sleep(0.5)
[sub_1]
[sub_2]
[sub_3]
[sub_4]
[a_1]
[a_2]
[a_3]
sos_run('sub')
[a_4]
[a_5]
''')
wf = script.workflow('a')
Base_Executor(wf).run()
def __init__(self, workflow=None, args=[], nested=False, config={}):
# we actually do not have our own workflow, everything is passed from ipython
# by nested = True we actually mean no new dictionary
if env.sig_mode is None:
env.sig_mode = 'ignore'
Base_Executor.__init__(self, workflow=workflow, args=args, nested=True, config=config)
env.__task_engine__ = 'interactive'
if env.sig_mode != 'ignore':
self.md5 = self.create_signature()
# We append to existing workflow files because some files are ignored and we
# still wants their information.
with open(os.path.join(env.exec_dir, '.sos', '{}.sig'.format(self.md5)), 'a') as sig:
sig.write('# workflow: {}\n'.format(self.workflow.name))
# script is None because it is entered from notebook
with open('__interactive__.sos', 'w') as script:
script.write(self.workflow.content.content)
sig.write('# script: {}\n'.format('__interactive__.sos'))
sig.write('# included: {}\n'.format(','.join(self.workflow.content.included)))
sig.write('# configuration: {}\n'.format(config.get('config_file', '')))
sig.write('# start time: {}\n'.format(time.strftime('%a, %d %b %Y %H:%M:%S +0000', time.gmtime())))
sig.write(self.sig_content)
sig.write('# runtime signatures\n')
def testTcsh(self):
'''Test action tcsh'''
if not shutil.which('tcsh'):
return
script = SoS_Script(r'''
[0]
tcsh:
foreach color (red orange yellow green blue)
echo $color
end
''')
wf = script.workflow()
Base_Executor(wf).run()
def testUndetermined(self):
'''Test DAG with undetermined input.'''
#
for filename in ('a.txt', 'd.txt'):
with open(filename, 'w') as tmp:
tmp.write('hey')
# input of step 3 is undertermined so
# it depends on all its previous steps.
script = SoS_Script('''
[C_1]
input: 'a.txt'
output: 'b.txt'
[C_2]
input: 'b.txt'
output: 'c.txt'
[C_3]
input: dynamic('*.txt')
output: 'd.txt'
[C_4]
depends: 'd.txt'
output: 'e.txt'
''')
wf = script.workflow()
dag = Base_Executor(wf).initialize_dag()
dag.show_nodes()
#dag.write_dot('a.dot')
self.assertDAG(dag,
'''
strict digraph "" {
C_1;
C_4;
C_2;
C_3;
C_1 -> C_2;
C_2 -> C_3;
C_3 -> C_4;
}
''')
#
# output of step
#
script = SoS_Script('''
[C_1]
input: 'a.txt'
output: 'b.txt'
[C_2]
input: 'b.txt'
output: 'c.txt'
[C_3]
input: dynamic('*.txt')
[C_4]
depends: 'd.txt'
output: 'e.txt'
''')
wf = script.workflow()
dag = Base_Executor(wf).initialize_dag()
self.assertDAG(dag,
'''
strict digraph "" {
C_1;
C_4;
C_2;
C_3;
C_1 -> C_2;
C_2 -> C_3;
C_3 -> C_4;
}
''')
for filename in ('a.txt', 'd.txt'):
os.remove(filename)