def testActionScript(self):
'''Test action script'''
script = SoS_Script(r'''
[A_1]
script: interpreter='python'
with open('something.txt', 'w') as tmp:
tmp.write('something')
''')
wf = script.workflow()
Base_Executor(wf).run()
self.assertTrue(file_target('something.txt').target_exists())
with open('something.txt') as tmp:
self.assertEqual('something', tmp.read())
def testRemoteExecute(self):
if os.path.isfile('result_remote.txt'):
os.remove('result_remote.txt')
if os.path.isfile('local.txt'):
os.remove('local.txt')
with open('local.txt', 'w') as w:
w.write('something')
self.assertEqual(
subprocess.call('sos push local.txt -c ~/docker.yml --to docker',
shell=True), 0)
with open('test_remote.sos', 'w') as tr:
tr.write('''
[10]
input: 'local.txt'
output: 'result_remote.txt'
task:
run:
cp local.txt result_remote.txt
echo 'adf' >> 'result_remote.txt'
''')
self.assertEqual(
subprocess.call(
'sos run test_remote.sos -c ~/docker.yml -r docker -s force',
shell=True), 0)
self.assertFalse(file_target('result_remote.txt').target_exists())
#self.assertEqual(subprocess.call('sos preview result_remote.txt -c ~/docker.yml -r docker', shell=True), 0)
#self.assertNotEqual(subprocess.call('sos preview result_remote.txt', shell=True), 0)
self.assertEqual(
subprocess.call(
'sos pull result_remote.txt -c ~/docker.yml --from docker',
shell=True), 0)
self.assertTrue(file_target('result_remote.txt').target_exists())
#self.assertEqual(subprocess.call('sos preview result_remote.txt', shell=True), 0)
with open('result_remote.txt') as w:
content = w.read()
self.assertTrue('something' in content, 'Got {}'.format(content))
self.assertTrue('adf' in content, 'Got {}'.format(content))
def test_concurrent_task(self):
'''Test submitting tasks from concurrent substeps'''
for f in [f'con_{x}.txt' for x in range(5)]:
if file_target(f).exists():
file_target(f).unlink()
script = SoS_Script('''
[10]
input: for_each={'i': range(5)}
output: f'con_{i}.txt'
task:
run: expand=True
echo {i} > {_output}
''')
wf = script.workflow()
Base_Executor(
wf, config={
'sig_mode': 'force',
'default_queue': 'localhost'
}).run()
for f in [f'con_{x}.txt' for x in range(5)]:
self.assertTrue(file_target(f).exists())
def testFileSig(self):
'''test save and validate of file signature'''
with open('test_sig.txt', 'w') as ts:
ts.write('ba')
a = file_target('test_sig.txt')
a.write_sig()
self.assertTrue(a.validate())
#
a.zap()
self.assertTrue(a.validate())
with open('test_sig.txt', 'w') as ts:
ts.write('bac')
self.assertFalse(a.validate())
def testSharedVarInForEach(self):
self.touch(['1.txt', '2.txt'])
for file in ('1.out', '2.out', '1.out2', '2.out2'):
if file_target(file).exists():
file_target(file).unlink()
script = SoS_Script('''
[work_1: shared = {'data': 'step_output'}]
input: "1.txt", "2.txt", group_by = 'single', pattern = '{name}.{ext}'
output: expand_pattern('{_name}.out')
run: expand=True
touch {_output}
[work_2]
depends: sos_variable('data')
input: "1.txt", "2.txt", group_by = 'single', for_each = dict(data=data), pattern = '{name}.{ext}'
output: expand_pattern('{data}_{_name}.out2')
run: expand=True
touch {_output}
''')
wf = script.workflow()
Base_Executor(wf).run()
def testPassOfTargetSource(self):
'''Test passing of source information from step_output'''
script = SoS_Script('''
[1]
output: 'a.txt'
_output.touch()
[2]
assert step_input.sources == ['1']
''')
wf = script.workflow()
Base_Executor(wf).run()
#
script = SoS_Script('''
[1]
input: for_each={'i': range(2)}
output: 'a.txt', 'b.txt', group_by=1
_output.touch()
[2]
assert step_input.sources == ['1', '1']
''')
wf = script.workflow()
Base_Executor(wf).run()
#
file_target('c.txt').touch()
script = SoS_Script('''
[1]
input: for_each={'i': range(2)}
output: 'a.txt', 'b.txt', group_by=1
_output.touch()
[2]
input: 'c.txt'
assert step_input.sources == ['2']
''')
wf = script.workflow()
Base_Executor(wf).run()
def testLiteralConnection(self):
'''Testing the connection of steps with by variables.'''
for f in ['A1.txt']:
if file_target(f).exists():
file_target(f).unlink()
#
# A1 introduces a shared variable ss, A3 depends on ss but not A2
#
script = SoS_Script('''
[A_1: shared='p']
run:
touch 'A1.txt'
p = 'A1.txt'
[A_2]
input: None
run:
sleep 0
[A_3]
input: p
depends: sos_variable('p')
run:
sleep 0
[A_4]
input: p
depends: sos_variable('p')
run:
sleep 0
[A_5]
input: dynamic(p)
depends: sos_variable('p')
''')
wf = script.workflow('A')
dag = Base_Executor(wf).initialize_dag()
self.assertDAG(
dag, '''
strict digraph "" {
A_1;
A_4;
A_2;
A_3;
A_5;
A_1 -> A_4;
A_1 -> A_3;
A_1 -> A_5;
}
''')
env.max_jobs = 3
Base_Executor(wf).run()
for f in ['A1.txt']:
self.assertTrue(file_target(f).target_exists())
file_target(f).unlink()
def test_rmarkdown(self):
'''Test action Rmarkdown'''
if file_target('myreport.html').exists():
file_target('myreport.html').unlink()
script = SoS_Script(r'''
[10]
report:
## Some random figure
Generated by matplotlib
[100]
# generate report
output: 'myreport.html'
Rmarkdown(output=_output[0])
''')
wf = script.workflow()
Base_Executor(wf, config={'report_output': 'report.md'}).run()
self.assertTrue(os.path.isfile('myreport.html'))
#
file_target('myreport.html').unlink()
def testStepWithMultipleOutput(self):
'''Test addition of steps with multiple outputs. It should be added only once'''
script = SoS_Script('''
[test_1: provides=['{}.txt'.format(i) for i in range(10)]]
output: ['{}.txt'.format(i) for i in range(10)]
run:
touch {output}
[test_2: provides=['{}.txt'.format(i) for i in range(10, 20)]]
depends: ['{}.txt'.format(i) for i in range(10)]
output: ['{}.txt'.format(i) for i in range(10, 20)]
run:
touch {output}
[default]
depends: ['{}.txt'.format(i) for i in range(10, 20)]
''')
wf = script.workflow()
Base_Executor(wf, config={'output_dag': 'test.dot'}).initialize_dag()
with open('test.dot') as dot:
lc = len(dot.readlines())
self.assertTrue(lc, 6)
file_target('test.dot').unlink()
def testSignatureWithSharedVariable(self):
'''Test restoration of signature from variables.'''
file_target('a.txt').remove('both')
# shared
script = SoS_Script(r"""
[0: shared='a']
output: 'a.txt'
run:
sleep 3
touch a.txt
a= 5
[1]
print(a)
""")
# alias should also be recovered.
wf = script.workflow('default')
Base_Executor(wf).run()
# rerun
Base_Executor(wf).run()
file_target('a.txt').remove('both')
def test_remote_exec_workdir_named_path(clear_now_and_after):
clear_now_and_after(file_target("#home/wd/result_workdir_named_path.txt"))
execute_workflow(
"""
output: '#home/wd/result_workdir_named_path.txt'
task: workdir='/root'
sh: expand=True
echo Output: {_output} > {_output}
echo PWD: `pwd`. >> {_output}
""",
options={
"config_file": "~/docker.yml",
"default_queue": "docker",
"sig_mode": "force",
},
)
assert file_target(
"#home/wd/result_workdir_named_path.txt").target_exists()
with open(file_target("#home/wd/result_workdir_named_path.txt")) as res:
result = res.read()
assert "Output: /root/wd/result_workdir_named_path.txt" in result
assert "PWD: /root." in result
def test_remote_exec(clear_now_and_after):
clear_now_and_after("result_exec.txt")
root_dir = "/root/build" if "TRAVIS" in os.environ else "/root"
execute_workflow(
"""
output: 'result_exec.txt'
task:
sh: expand=True
echo Output: {_output} > {_output}
echo PWD: `pwd`. >> {_output}
""",
options={
"config_file": "~/docker.yml",
"default_queue": "docker",
"sig_mode": "force",
},
)
assert file_target("result_exec.txt").target_exists()
with open(file_target("result_exec.txt")) as res:
result = res.read()
assert "Output: result_exec.txt" in result
assert f"PWD: {root_dir}/vatlab/sos/test." in result
def testSignatureWithVars(self):
'''Test revaluation with variable change'''
self.touch(('a1.out', 'a2.out'))
for f in ('b1.out', 'b2.out'):
if file_target(f).exists():
file_target(f).unlink()
script = SoS_Script('''
parameter: DB = {'input': ['a1.out'], 'output': ['b1.out']}
parameter: input_file = DB['input']
parameter: output_file = DB['output']
[2]
input: input_file, group_by = 1
output: output_file[_index]
run: expand=True
touch {_output}
''')
wf = script.workflow()
res = Base_Executor(wf).run()
self.assertEqual(res['__completed__']['__step_completed__'], 1)
ts = os.path.getmtime('b1.out')
#
script = SoS_Script('''
parameter: DB = {'input': ['a1.out', 'a2.out'], 'output': ['b1.out', 'b2.out']}
parameter: input_file = DB['input']
parameter: output_file = DB['output']
[2]
input: input_file, group_by = 1
output: output_file[_index]
run: expand=True
touch {_output}
''')
wf = script.workflow()
res = Base_Executor(wf).run()
self.assertEqual(res['__completed__']['__step_completed__'], 0.5)
self.assertEqual(ts, os.path.getmtime('b1.out'))
def testSoSStep(self):
'''Test target sos_step'''
for file in ['t1.txt', 't2.txt', '5.txt', '10.txt', '20.txt']:
file_target(file).remove('both')
script = SoS_Script('''
[t1]
run:
touch t1.txt
[t2: provides='t2.txt']
depends: sos_step('t1')
run:
touch t2.txt
[5]
run:
touch 5.txt
[10]
depends: sos_step('t2')
run:
touch 10.txt
[20]
depends: sos_step('t1')
task:
run:
touch 20.txt
''')
wf = script.workflow()
env.config['sig_mode'] = 'force'
# this should be ok.
Base_Executor(wf).run()
for file in ['t1.txt', 't2.txt', '5.txt', '10.txt', '20.txt']:
self.assertTrue(
file_target(file).target_exists(), file + ' should exist')
file_target(file).remove('both')
def testSignatureAfterRemovalOfFiles(self):
'''test action shrink'''
if os.path.isfile('largefile.txt'):
os.remove('largefile.txt')
script = SoS_Script(r'''
[10]
# generate a file
output: 'largefile.txt'
run: expand='${ }'
for x in {1..1000}
do
echo $x >> ${output}
done
''')
wf = script.workflow()
Base_Executor(wf).run()
# sleep 3
# rerun, because this is the final target, it has to be
# re-generated
os.remove('largefile.txt')
Base_Executor(wf).run()
self.assertTrue(os.path.isfile('largefile.txt'))
#
# we discard the signature, the step would still be
# skipped because file signature will be calculated
# during verification
file_target('largefile.txt').remove('signature')
Base_Executor(wf).run()
#
# now if we touch the file, it needs to be regenerated
with open('largefile.txt', 'a') as lf:
lf.write('something')
Base_Executor(wf).run()
file_target('largefile.txt').remove('both')
def testRegenerateReport(self):
'''Testing the regeneration of report once is needed. The problem
here is the 'input' parameter of report.'''
script = SoS_Script(r'''
[A_1]
output: 'a1.txt', 'a1.md'
run:
echo 'a1' >> a1.txt
report: output='a1.md'
a1
[A_2]
output: 'a2.txt', 'a2.md'
run:
echo 'a2' >> a2.txt
report: output='a2.md'
a2
[A_3]
input: 'a1.md', 'a2.md'
output: 'out.md'
report: input=['a1.md', 'a2.md'], output='out.md'
''')
wf = script.workflow()
Base_Executor(wf).run()
with open('a1.md') as a:
self.assertEqual(a.read(), 'a1\n\n')
with open('a2.md') as a:
self.assertEqual(a.read(), 'a2\n\n')
with open('out.md') as a:
self.assertEqual(a.read(), 'a1\n\na2\n\n')
for name in ('a1.md', 'a2.md', 'out.md'):
if file_target(name).exists():
file_target(name).unlink()
wf = script.workflow()
Base_Executor(wf).run()
def testLoopedNestedWorkflow(self):
# step will be looped
self.touch(['a.txt', 'b.txt'])
script = SoS_Script('''
if 'executed' not in locals():
executed = []
if 'inputs' not in locals():
inputs = []
[a_1:shared=['executed', 'inputs']]
executed.append(step_name)
output: _input[0] + '.a1'
inputs.append(_input)
run: expand=True
touch {_output}
[a_2:shared=['executed', 'inputs']]
executed.append(step_name)
output: _input[0] + '.a2'
inputs.append(_input)
run: expand=True
touch {_output}
[c:shared=['executed', 'inputs']]
executed.append(step_name)
input: 'a.txt', 'b.txt', group_by='single'
inputs.append(_input)
sos_run('a', shared=['executed', 'inputs'])
''')
wf = script.workflow('c')
Base_Executor(wf).run()
self.assertEqual(env.sos_dict['executed'],
['c', 'a_1', 'a_2', 'a_1', 'a_2'])
#self.assertEqual(env.sos_dict['inputs'], [['a.txt'], ['a.txt'], ['a.txt.a1'], ['b.txt'], ['b.txt'], ['b.txt.a1']])
for file in ('a.txt.a1', 'a.txt.a1.a2', 'b.txt.a1', 'b.txt.a1.a2'):
if file_target(file).exists():
file_target(file).unlink()
def testReverseSharedVariable(self):
'''Test shared variables defined in auxiliary steps'''
file_target('a.txt').remove('both')
script = r'''
%run B
[A: shared='b', provides='a.txt']
b = 1
run:
touch a.txt
[B_1]
depends: 'a.txt'
[B_2]
print(b)
'''
with sos_kernel() as kc:
iopub = kc.iopub_channel
execute(kc=kc, code=script)
wait_for_idle(kc)
execute(kc=kc, code="b")
res = get_result(iopub)
self.assertEqual(res, 1)
def test_zap(self):
"""Test zap"""
with open("testzap.txt", "w") as sf:
sf.write("some text")
path("testzap.txt").zap()
self.assertTrue(os.path.isfile("testzap.txt.zapped"))
self.assertFalse(os.path.isfile("testzap.txt"))
# re-zap is ok
file_target("testzap.txt").zap()
self.assertTrue(os.path.isfile("testzap.txt.zapped"))
self.assertFalse(os.path.isfile("testzap.txt"))
# non-existent file
os.remove("testzap.txt.zapped")
self.assertRaises(FileNotFoundError, path("testzap.txt").zap)
#
with open("testzap.txt", "w") as sf:
sf.write("some text")
with open("testzap1.txt", "w") as sf:
sf.write("some text")
paths("testzap.txt", "testzap1.txt").zap()
self.assertTrue(os.path.isfile("testzap.txt.zapped"))
self.assertFalse(os.path.isfile("testzap.txt"))
self.assertTrue(os.path.isfile("testzap1.txt.zapped"))
self.assertFalse(os.path.isfile("testzap1.txt"))
#
os.remove("testzap.txt.zapped")
os.remove("testzap1.txt.zapped")
with open("testzap.txt", "w") as sf:
sf.write("some text")
with open("testzap1.txt", "w") as sf:
sf.write("some text")
sos_targets(["testzap.txt", "testzap1.txt"]).zap()
self.assertTrue(os.path.isfile("testzap.txt.zapped"))
self.assertFalse(os.path.isfile("testzap.txt"))
self.assertTrue(os.path.isfile("testzap1.txt.zapped"))
self.assertFalse(os.path.isfile("testzap1.txt"))
def testExecutionLock(self):
'''Test execution lock of two processes'''
with open('lock.sos', 'w') as lock:
lock.write(r'''
import time
[A_1]
output: 'a.txt'
with open('a.txt', 'w') as txt:
txt.write('A1\n')
# A1 and A2 are independent
[A_2]
input: None
output: 'b.txt'
with open('b.txt', 'w') as txt:
txt.write('A2\n')
''')
ret1 = subprocess.Popen('sos run lock -j1', shell=True)
ret2 = subprocess.Popen('sos run lock -j1', shell=True)
ret1.wait()
ret2.wait()
# two processes execute A_1 and A_2 separately, usually
# takes less than 5 seconds
file_target('lock.sos').remove('both')
def test_zap(self):
'''Test zap'''
with open('testzap.txt', 'w') as sf:
sf.write('some text')
path('testzap.txt').zap()
self.assertTrue(os.path.isfile('testzap.txt.zapped'))
self.assertFalse(os.path.isfile('testzap.txt'))
# re-zap is ok
file_target('testzap.txt').zap()
self.assertTrue(os.path.isfile('testzap.txt.zapped'))
self.assertFalse(os.path.isfile('testzap.txt'))
# non-existent file
os.remove('testzap.txt.zapped')
self.assertRaises(FileNotFoundError, path('testzap.txt').zap)
#
with open('testzap.txt', 'w') as sf:
sf.write('some text')
with open('testzap1.txt', 'w') as sf:
sf.write('some text')
paths('testzap.txt', 'testzap1.txt').zap()
self.assertTrue(os.path.isfile('testzap.txt.zapped'))
self.assertFalse(os.path.isfile('testzap.txt'))
self.assertTrue(os.path.isfile('testzap1.txt.zapped'))
self.assertFalse(os.path.isfile('testzap1.txt'))
#
os.remove('testzap.txt.zapped')
os.remove('testzap1.txt.zapped')
with open('testzap.txt', 'w') as sf:
sf.write('some text')
with open('testzap1.txt', 'w') as sf:
sf.write('some text')
sos_targets(['testzap.txt', 'testzap1.txt']).zap()
self.assertTrue(os.path.isfile('testzap.txt.zapped'))
self.assertFalse(os.path.isfile('testzap.txt'))
self.assertTrue(os.path.isfile('testzap1.txt.zapped'))
self.assertFalse(os.path.isfile('testzap1.txt'))
def testOptionWorkdir(self):
'''Test option workdir of tasks'''
if not os.path.isdir('temp_wdr'):
os.mkdir('temp_wdr')
with open(os.path.join('temp_wdr', 'a.txt'), 'w') as tmp:
tmp.write('hello')
script = SoS_Script(r'''
[A_1]
run: workdir='temp_wdr'
cp -f a.txt a2.txt
''')
wf = script.workflow()
Base_Executor(wf).run()
self.assertTrue(file_target(os.path.join('temp_wdr', 'a2.txt')).target_exists())
with open(os.path.join('temp_wdr', 'a.txt')) as tmp:
self.assertEqual('hello', tmp.read())
def testOptionWorkdir(temp_factory):
'''Test option workdir of tasks'''
temp_factory(dir='temp_wdr')
with open(os.path.join('temp_wdr', 'a.txt'), 'w') as tmp:
tmp.write('hello')
execute_workflow(r'''
[A_1]
run: workdir='temp_wdr'
cp -f a.txt a2.txt
''')
assert file_target(os.path.join('temp_wdr', 'a2.txt')).target_exists()
with open(os.path.join('temp_wdr', 'a.txt')) as tmp:
assert 'hello' == tmp.read()
def testRemoteTS(self):
if os.path.exists('ar.txt'):
os.remove('ar.txt')
with open('remote_ts.sos', 'w') as rt:
rt.write('''
[10]
task:
sh:
echo "I am done" >> ar.txt
''')
ret = subprocess.call('sos run remote_ts -c ~/docker.yml -q ts',
shell=True)
self.assertTrue(ret == 0)
ret = subprocess.call('sos pull ar.txt -c ~/docker.yml --from ts',
shell=True)
self.assertTrue(ret == 0)
self.assertTrue(file_target('ar.txt').target_exists())
def test_forward_style_depend(clear_now_and_after, temp_factory):
'''Test the execution of forward-style workflow with undtermined dependency'''
clear_now_and_after('a.txt.bak')
temp_factory('a.txt')
execute_workflow('''
[10]
input: 'a.txt'
output: f"{_input}.bak"
run: expand=True
cp {_input} {_output}
[20]
depends: "a.txt.bak"
run: expand=True
ls {_depends}
''')
assert file_target('a.txt.bak').target_exists()
def test_file_sig(clear_now_and_after):
'''test save and validate of file signature'''
clear_now_and_after('test_sig.txt')
with open('test_sig.txt', 'w') as ts:
ts.write('ba')
a = file_target('test_sig.txt')
a.write_sig()
assert a.validate()
#
a.zap()
assert a.validate()
with open('test_sig.txt', 'w') as ts:
ts.write('bac')
assert not a.validate()
def testSharedOption(self):
'''Test shared option of task'''
for f in ('a.txt', 'a100.txt'):
if file_target(f).exists():
file_target(f).unlink()
script = SoS_Script('''
[10: shared = 'a']
output: 'a.txt'
task: shared={'a': 'int(open("a.txt").read())'}
run:
echo 100 > a.txt
[20]
run: expand=True
touch a{a}.txt
''')
wf = script.workflow()
Base_Executor(wf, config={'sig_mode': 'force'}).run()
self.assertTrue(os.path.isfile("a100.txt"))
# sequence of var or mapping
for f in ('a.txt', 'a100.txt'):
if file_target(f).exists():
file_target(f).unlink()
script = SoS_Script('''
[10: shared = ['a', 'b']]
output: 'a.txt'
task: shared=[{'a': 'int(open("a.txt").read())'}, 'b']
b = 20
run:
echo 100 > a.txt
[20]
run: expand=True
touch a{a}_{b}.txt
''')
wf = script.workflow()
Base_Executor(wf, config={'sig_mode': 'force'}).run()
self.assertTrue(os.path.isfile("a100_20.txt"))
script = SoS_Script('''
[10 (simulate): shared=['rng', 'step_rng']]
input: for_each={'i': range(5)}
task: shared='rng'
print(f"{i}")
import random
rng = random.randint(1, 1000)
''')
wf = script.workflow()
Base_Executor(wf).run()
var = env.sos_dict['rng']
self.assertTrue(isinstance(var, int))
self.assertTrue(isinstance(env.sos_dict['step_rng'], list))
self.assertEqual(env.sos_dict['step_rng'][-1], var)
def test_remote_exec_workdir_wo_named_path(clear_now_and_after):
clear_now_and_after(file_target("result_workdir_wo_named.txt"))
with pytest.raises(Exception):
execute_workflow(
"""
output: 'result_workdir_wo_named.txt'
task: workdir='/other'
sh: expand=True
echo Output: {_output} > {_output}
echo PWD: `pwd`. >> {_output}
""",
options={
"config_file": "~/docker.yml",
"default_queue": "docker",
"sig_mode": "force",
},
)
def test_report_3(clear_now_and_after):
'''Test action report'''
clear_now_and_after('report.txt')
#
execute_workflow(r'''
[A_1]
run: output='a.txt'
echo something > a.txt
[A_2]
run: output='b.txt'
echo something else > b.txt
[A_3]
report(input=['a.txt', 'b.txt'], output='out.txt')
''')
for name in ('a.txt', 'b.txt', 'out.txt'):
assert (file_target(name).target_exists())
def testLocalTS(self):
#
if os.path.exists('a.txt'):
os.remove('a.txt')
script = SoS_Script('''
[10]
task:
sh:
echo "I am done" >> a.txt
''')
wf = script.workflow()
Base_Executor(wf,
config={
'config_file': '~/docker.yml',
'wait_for_task': True,
'default_queue': 'local_ts',
'sig_mode': 'force',
}).run()
self.assertTrue(file_target('a.txt').target_exists())