def testProgressBar(self):
'''Test progress bar'''
env.verbosity = 1
prog = ProgressBar('test', 100)
for i in range(100):
prog.update(i)
prog.done()
prog = ProgressBar('test', 100)
for i in range(20):
prog.progress(5)
prog.done()
#
script = SoS_Script('''
[1]
[2]
[3]
[4]
[5]
''')
wf = script.workflow()
Base_Executor(wf).run()
# progress bar with nested workflow
script = SoS_Script('''
import time
time.sleep(0.5)
[sub_1]
[sub_2]
[sub_3]
[sub_4]
[a_1]
[a_2]
[a_3]
sos_run('sub')
[a_4]
[a_5]
''')
wf = script.workflow('a')
Base_Executor(wf).run()
def testTcsh(self):
'''Test action tcsh'''
if not shutil.which('tcsh'):
return
script = SoS_Script(r'''
[0]
tcsh:
foreach color (red orange yellow green blue)
echo $color
end
''')
wf = script.workflow()
Base_Executor(wf).run()
def testTextRepr(self):
# the " as the last character can lead to problems...
script = SoS_Script('''
run:
echo "Hi, This is from bash"''')
wf = script.workflow()
Base_Executor(wf).run()
#
for text in ('"""a"""', '"b"',
r'"""\na\\nb"""', r"'''a\nb'''",
""" "a'\\"='" """):
script = SoS_Script(r'''
a = 1
python:
with open('tmp.txt', 'w') as tmp:
tmp.write({} + '{}')
k = """b"""'''.format(text, '${a}')
)
wf = script.workflow()
Base_Executor(wf).run()
with open('tmp.txt') as tmp:
self.assertEqual(tmp.read(), eval(text) + '1')
os.remove('tmp.txt')
def testMixedTabAndSpace(self):
'''Test handling of mixed tab and space'''
script = SoS_Script('''
[1: shared=['a', 'b', 'c']]
if True:
a = 1
\tb = 2
\tc= 3
''')
wf = script.workflow()
Base_Executor(wf).run()
self.assertEqual(env.sos_dict['a'], 1)
self.assertEqual(env.sos_dict['b'], 2)
self.assertEqual(env.sos_dict['c'], 3)
def testStopIf(self):
'''Test action stop_if'''
script = SoS_Script(r'''
[0: shared='result']
rep = range(20)
result = []
input: for_each='rep'
stop_if(_rep > 10)
result.append(_rep)
''')
wf = script.workflow()
Base_Executor(wf).run()
self.assertEqual(env.sos_dict['result'],
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
def testSectionActions(self):
'''Test actions of sections'''
SoS_Script(
"""
[0]
func('''
multiline
string''', with_option=1
)
""")
self.assertRaises(ParsingError, SoS_Script,
'''
[0]
func(
''')
def testDynamicNestedWorkflow(self):
#
# Because we are not sure which workflows would be executed
# until run time, the DAG should not contain nested workflow
# until runtime.
#
for f in [
'B0.txt', 'B0.txt.p', 'B1.txt', 'B1.txt.p', 'B2.txt',
'B2.txt.p'
]:
FileTarget(f).remove('both')
#
# A1 <- P <- B
# A1 <- P <- B
# A2
#
# ALL calls A and B with parameter
#
script = SoS_Script('''
[A_1]
parameter: num = 2
input: "B${num}.txt.p"
[B: provides='B{num}.txt']
sh:
touch 'B${num}.txt'
[P: provides='{filename}.p']
input: filename
sh:
touch ${output}
[ALL]
for num in range(3):
sos_run('A')
''')
# the workflow should call step K for step C_2, but not C_3
wf = script.workflow('ALL')
Base_Executor(wf).run()
for f in [
'B0.txt', 'B0.txt.p', 'B1.txt', 'B1.txt.p', 'B2.txt',
'B2.txt.p'
]:
self.assertTrue(FileTarget(f).exists())
FileTarget(f).remove('both')
def testParallelExecution(self):
'''Test basic parallel execution'''
'''
A1 <- None
A2 <- B2
'''
for f in ['A1.txt', 'B2.txt', 'A2.txt']:
FileTarget(f).remove('both')
script = SoS_Script('''
[A_1]
output: 'A1.txt'
sh:
sleep 3
touch A1.txt
[A_2]
input: 'B2.txt'
output: 'A2.txt'
sh:
sleep 3
touch A2.txt
[B: provides='B2.txt']
output: 'B2.txt'
sh:
touch B2.txt
''')
# the workflow should call step K for step C_2, but not C_3
wf = script.workflow()
dag = Base_Executor(wf).initialize_dag()
self.assertDAG(
dag, '''
strict digraph "" {
A_1;
A_2;
"B ['B2.txt']";
"B ['B2.txt']" -> A_2;
}
''')
env.max_jobs = 4
st = time.time()
#env.verbosity = 4
MP_Executor(wf).run()
self.assertLess(time.time() - st, 4)
for f in ['A1.txt', 'B2.txt', 'A2.txt']:
FileTarget(f).remove('both')
def testDockerImage(self):
'''Test docker_image option'''
script = SoS_Script(r'''
[0]
fastq_files = glob.glob('data/*.fastq')
input_volume = os.path.dirname(fastq_files[0])
output_volume = os.getcwd()
run: docker_image='compbio/ngseasy-fastqc:1.0-r001',
volumes=["${input_volume}:/input_data", "${output_volume}:/output_data"]
ls -l /input_data
/usr/local/bin/fastqc /input_data/*.fastq --outdir /output_data
''')
wf = script.workflow()
Base_Executor(wf).run()
def testAnalyzeSection(self):
'''Test analysis of sections (statically)'''
script = SoS_Script('''
g1 = 'a'
g2 = 1
parameter: p1 = 5
parameter: infiles = 'a.txt'
[A_1: shared='b']
b = p1 + 2
input: infiles
output: None
c = 5
[A_2]
b = [1, 2, 3]
input: for_each='b'
depends: 'some.txt', executable('ls')
import time
import random
r = random.randint(1, 5)
time.sleep(r)
[A_3]
input: None
print(p1)
''')
wf = script.workflow('A')
for section in wf.sections:
res = analyze_section(section)
if section.names[0][1] == '1':
self.assertTrue(isinstance(res['step_input'], Undetermined))
self.assertEqual(res['step_depends'], [])
self.assertEqual(res['step_output'], [])
self.assertEqual(res['environ_vars'], {'p1', 'infiles'})
self.assertEqual(res['signature_vars'], {'c'})
self.assertEqual(res['changed_vars'], {'b'})
elif section.names[0][1] == '2':
self.assertEqual(res['step_input'], [])
self.assertEqual(res['step_depends'], ['some.txt', executable('ls')])
self.assertTrue(isinstance(res['step_output'], Undetermined))
# for_each will not be used for DAG
self.assertEqual(res['environ_vars'], {'for_each'})
self.assertEqual(res['signature_vars'], {'import', 'r', 'time', 'random'})
self.assertEqual(res['changed_vars'], set())
def testRubyInDocker(self):
'''Test action ruby in docker environment'''
script = SoS_Script(r'''
[0]
ruby: docker_image='ruby'
line1 = "Cats are smarter than dogs";
line2 = "Dogs also like meat";
if ( line1 =~ /Cats(.*)/ )
puts "Line1 contains Cats"
end
if ( line2 =~ /Cats(.*)/ )
puts "Line2 contains Dogs"
end
''')
wf = script.workflow()
Base_Executor(wf).run()
def testSharedDependency(self):
#
# shared variable should introduce additional dependency
#
for f in ['A1.txt']:
FileTarget(f).remove('both')
#
# A1 introduces a shared variable ss, A3 depends on ss but not A2
#
script = SoS_Script('''
[A_1: shared='ss']
ss = 'A1'
[A_2]
input: None
sh:
sleep 3
[A_3]
input: None
import time
time.sleep(3)
with open("${ss}.txt", 'w') as tmp:
tmp.write('test')
''')
wf = script.workflow('A')
dag = Base_Executor(wf).initialize_dag()
self.assertDAG(
dag, '''
strict digraph "" {
A_3;
A_1;
A_2;
A_1 -> A_3;
}
''')
env.max_jobs = 3
st = time.time()
MP_Executor(wf).run()
self.assertLess(time.time() - st, 5)
for f in ['A1.txt']:
self.assertTrue(FileTarget(f).exists())
FileTarget(f).remove('both')
def testSkipStep(self):
'''Test the skip option to skip certain steps'''
script = SoS_Script('''
parameter: skip = 0
[0: shared={'a':'var'}, skip=skip==0]
var = 0
[1: shared={'b': 'var'}, skip=skip==1]
var = 1
''')
wf = script.workflow()
Base_Executor(wf, args=['--skip', '0']).run()
self.assertEqual(env.sos_dict['b'], 1)
#
Base_Executor(wf, args=['--skip', '1']).run()
self.assertEqual(env.sos_dict['a'], 0)
def testVariableTarget(self):
'''Test dependency caused by variable usage.'''
script = SoS_Script(r'''
[A: shared='b']
b = 1
[C: shared={'c':'k'}]
k = 2
[all: shared='p']
depends: sos_variable('c'), sos_variable('b')
p = c + b
''')
wf = script.workflow('all')
Base_Executor(wf).run()
self.assertTrue(env.sos_dict['p'], 3)
def testCombinedWorkflow(self):
'''Test the creation and execution of combined workfow'''
script = SoS_Script('''
a0 = 0
if 'executed' not in locals():
executed = []
parameter: a = a0 + 1
[a_1: shared='executed']
executed.append(step_name)
[a_2: shared='executed']
executed.append(step_name)
[a_3: shared='executed']
executed.append(step_name)
[a_4: shared='executed']
executed.append(step_name)
output: 'out_a_4'
[b_1: shared=['executed', 'input_b1']]
executed.append(step_name)
input_b1 = input
[b_2: shared='executed']
executed.append(step_name)
[b_3: shared='executed']
executed.append(step_name)
[b_4: shared='executed']
executed.append(step_name)
[c: shared='executed']
executed.append(step_name)
[d: shared='executed']
executed.append(step_name)
''')
wf = script.workflow('a+b')
Base_Executor(wf).dryrun()
self.assertEqual(env.sos_dict['executed'], ['a_1', 'a_2', 'a_3', 'a_4', 'b_1', 'b_2', 'b_3', 'b_4'])
self.assertEqual(env.sos_dict['a'], 1)
self.assertEqual(env.sos_dict['input_b1'], ['out_a_4'])
#
wf = script.workflow('a: 1-2 + a:4 + b:3-')
Base_Executor(wf).dryrun()
self.assertEqual(env.sos_dict['executed'], ['a_1', 'a_2', 'a_4',
'b_3', 'b_4'])
#
wf = script.workflow('a+c+d')
Base_Executor(wf).dryrun()
self.assertEqual(env.sos_dict['executed'], ['a_1', 'a_2', 'a_3', 'a_4', 'c_0', 'd_0'])
def testReverseSharedVariable(self):
'''Test shared variables defined in auxiliary steps'''
FileTarget('a.txt').remove('both')
script = SoS_Script(r'''
[A: shared='b', provides='a.txt']
b = 1
sh:
touch a.txt
[B_1]
depends: 'a.txt'
[B_2]
print(b)
''')
wf = script.workflow('B')
Base_Executor(wf).run()
self.assertTrue(env.sos_dict['b'], 1)
def testDockerImageFromFile(self):
'''Test docker_image load from a file.'''
# image from a saved file
script = SoS_Script(r'''
[0]
run: docker_image='blang/busybox-bash'
[1]
run:
docker save blang/busybox-bash > hello.tar
docker rmi -f blang/busybox-bash
[2]
run: docker_image='blang/busybox-bash', docker_file = 'hello.tar'
echo "a"
''')
wf = script.workflow()
Base_Executor(wf).run()
def testRuby(self):
'''Test action ruby'''
if not shutil.which('ruby'):
return True
script = SoS_Script(r'''
[0]
ruby:
line1 = "Cats are smarter than dogs";
line2 = "Dogs also like meat";
if ( line1 =~ /Cats(.*)/ )
puts "Line1 contains Cats"
end
if ( line2 =~ /Cats(.*)/ )
puts "Line2 contains Dogs"
end
''')
wf = script.workflow()
Base_Executor(wf).run()
def testDockerBuild(self):
'''Test action docker build'''
script = SoS_Script(r'''
[0]
docker_build: tag='test/docker_build'
#
# Super simple example of a Dockerfile
#
FROM ubuntu:latest
MAINTAINER Andrew Odewahn "*****@*****.**"
RUN apt-get update
RUN apt-get install -y python python-pip wget
RUN pip install Flask
WORKDIR /home
''')
wf = script.workflow()
Base_Executor(wf).run()
def testLongerCode(self):
'''Test definition of classes (with intermediate newlines) in step.'''
script = SoS_Script('''# first block
[0: shared='b']
class A:
def __init__(self):
pass
# the newline above should be fine because SoS treat this as
# regular lines
def __call__(self):
return 0
b = A()()
''')
wf = script.workflow()
Base_Executor(wf).run()
self.assertEqual(env.sos_dict['b'], 0)
def testSoSRun(self):
'''Test action sos_run with keyword parameters'''
for f in ['0.txt', '1.txt']:
FileTarget(f).remove('both')
script = SoS_Script(r'''
[A]
parameter: num=5
sh:
touch ${num}.txt
[batch]
for k in range(2):
sos_run('A', num=k)
''')
env.verbosity = 3
wf = script.workflow('batch')
Base_Executor(wf).run()
for f in ['0.txt', '1.txt']:
self.assertTrue(FileTarget(f).exists())
FileTarget(f).remove('both')
def testIncludeWithNamespace(self):
'''Test include a workflow that uses variables from its own global module'''
self.touch(['a.txt', 'b.txt'])
#
with open('inc.sos', 'w') as sos:
sos.write('''
# test sos script
# global definition
parameter: parB = 10
[A_1]
a = parB + 1
''')
script = SoS_Script('''
%include inc
''')
wf = script.workflow('inc.A')
Base_Executor(wf).dryrun()
def testIfElse(self):
'''Test if/elif/else/endif structural directive'''
# no matching %endif
self.assertRaises(ParsingError, SoS_Script, '''
%if 1
a = 1
%else
a=2
''')
# no if for else
self.assertRaises(ParsingError, SoS_Script, '''
%else
a=2
''')
# no conditon for if
self.assertRaises(ParsingError, SoS_Script, '''
%if
a=2
%endif
''')
# no conditon for elif
self.assertRaises(ParsingError, SoS_Script, '''
%if 1
%elif
a=2
%endif
[0]
''')
# test if else
script = SoS_Script('''
%if 0
a = 1
%else
a = 2
%endif
[0]
''')
wf = script.workflow()
Base_Executor(wf).dryrun()
self.assertEqual(env.sos_dict['a'], 2)
def testAuxiliarySteps(self):
script = SoS_Script('''
[K: provides='{name}.txt']
output: "${name}.txt"
sh:
touch '${name}.txt'
[C_2]
input: 'b.txt'
output: 'c.txt'
sh:
touch c.txt
[C_3]
input: 'a.txt'
''')
# a.txt exists and b.txt does not exist
with open('a.txt', 'w') as atfile:
atfile.write('garbage')
if os.path.isfile('b.txt'):
os.remove('b.txt')
# the workflow should call step K for step C_2, but not C_3
wf = script.workflow()
dag = Base_Executor(wf).initialize_dag()
#dag.write_dot('a.dot')
#dag.show_nodes()
self.assertDAG(
dag, '''
strict digraph "" {
"K ['b.txt']";
C_3;
C_2;
"K ['b.txt']" -> C_2;
}
''')
def testPandoc(self):
'''Test action pandoc'''
if not shutil.which('pandoc'):
return
script = SoS_Script(r'''
[10]
report:
## Some random figure
Generated by matplotlib
[100]
# generate report
output: 'myreport.html'
pandoc(output=_output[0], to='html')
''')
wf = script.workflow()
Base_Executor(wf).run()
self.assertTrue(os.path.isfile('myreport.html'))
#
FileTarget('myreport.html').remove('both')
def testCycle(self):
'''Test cycle detection of DAG'''
#
# A.txt --> B.txt
#
# B.txt --> C.txt
#
# C.txt --> A.txt
#
script = SoS_Script('''
[A_1]
input: 'A.txt'
output: 'B.txt'
[A_2]
output: 'C.txt'
[A_3]
output: 'A.txt'
''')
# the workflow should call step K for step C_2, but not C_3
wf = script.workflow()
self.assertRaises(RuntimeError, Base_Executor(wf).initialize_dag)
def testDownload(self):
'''Test download of resources'''
if not os.path.isdir('tmp'):
os.makedirs('tmp')
#
for name in [
'hapmap_ASW_freq.ann', 'hapmap_ASW_freq-hg18_20100817.DB.gz',
'hapmap_CHB_freq.ann', 'vt_quickStartGuide.tar.gz'
]:
if os.path.isfile(os.path.join('tmp', name)):
os.remove(os.path.join('tmp', name))
# test decompress tar.gz file
script = SoS_Script(r'''
[0]
download(['http://bioinformatics.mdanderson.org/Software/VariantTools/repository/snapshot/vt_quickStartGuide.tar.gz'],
dest_dir='tmp', decompress=True)
''')
wf = script.workflow()
Base_Executor(wf).run()
self.assertTrue(os.path.isfile('tmp/snapshot.proj'))
self.assertTrue(os.path.isfile('tmp/snapshot_genotype.DB'))
#
# testing the download of single file
#
script = SoS_Script(r'''
[0]
download: dest_file='tmp/test.ann'
http://bioinformatics.mdanderson.org/Software/VariantTools/repository/annoDB/hapmap_ASW_freq.ann
''')
wf = script.workflow()
Base_Executor(wf).run()
self.assertTrue(os.path.isfile('tmp/test.ann'))
# test option dest_dir
script = SoS_Script(r'''
[0]
download: dest_dir='tmp'
http://bioinformatics.mdanderson.org/Software/VariantTools/repository/annoDB/hapmap_ASW_freq.ann
''')
wf = script.workflow()
Base_Executor(wf).run()
self.assertTrue(os.path.isfile('tmp/hapmap_ASW_freq.ann'))
#
# this will take a while
script = SoS_Script(r'''
[0]
download: dest_dir='tmp', decompress=True
http://bioinformatics.mdanderson.org/Software/VariantTools/repository/annoDB/non-existing.gz
http://bioinformatics.mdanderson.org/Software/VariantTools/repository/annoDB/hapmap_ASW_freq.ann
http://bioinformatics.mdanderson.org/Software/VariantTools/repository/annoDB/hapmap_ASW_freq-hg18_20100817.DB.gz
http://bioinformatics.mdanderson.org/Software/VariantTools/repository/annoDB/hapmap_CHB_freq.ann
''')
start = time.time()
wf = script.workflow()
self.assertRaises(ExecuteError, Base_Executor(wf).run)
self.assertTrue(os.path.isfile('tmp/hapmap_ASW_freq-hg18_20100817.DB'))
self.assertGreater(time.time() - start, 3)
# this will be fast
start = time.time()
wf = script.workflow()
self.assertRaises(ExecuteError, Base_Executor(wf).run)
self.assertLess(time.time() - start, 3)
#
# test decompress tar.gz file
script = SoS_Script(r'''
[0]
download: dest_dir='tmp', decompress=True
http://bioinformatics.mdanderson.org/Software/VariantTools/repository/programs/SKAT_0.82.tar.gz
''')
wf = script.workflow()
Base_Executor(wf).run()
#
shutil.rmtree('tmp')
def testTarget(self):
'''Test executing only part of a workflow.'''
#
for f in [
'A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt',
'C1.txt', 'C3.txt', 'C4.txt'
]:
FileTarget(f).remove('both')
#
# A1 <- B1 <- B2 <- B3
# |
# |
# \/
# A2 <- B2 <- C1 <- C2 <- C4
# C3
#
script = SoS_Script('''
[A_1]
input: 'B1.txt'
output: 'A1.txt'
sh:
touch A1.txt
[A_2]
depends: 'B2.txt'
sh:
touch A2.txt
[B1: provides='B1.txt']
depends: 'B2.txt'
sh:
touch B1.txt
[B2: provides='B2.txt']
depends: 'B3.txt', 'C1.txt'
sh:
touch B2.txt
[B3: provides='B3.txt']
sh:
touch B3.txt
[C1: provides='C1.txt']
depends: 'C2.txt', 'C3.txt'
sh:
touch C1.txt
[C2: provides='C2.txt']
depends: 'C4.txt'
sh:
touch C2.txt
[C3: provides='C3.txt']
depends: 'C4.txt'
sh:
touch C3.txt
[C4: provides='C4.txt']
sh:
touch C4.txt
''')
# the workflow should call step K for step C_2, but not C_3
wf = script.workflow()
#
# test 1, we only need to generate target 'B1.txt'
dag = Base_Executor(wf).initialize_dag(targets=['B1.txt'])
# note that A2 is no longer mentioned
self.assertDAG(
dag, '''
strict digraph "" {
"B3 ['B3.txt']";
"C4 ['C4.txt']";
"C2 ['C2.txt']";
"C1 ['C1.txt']";
"B1 ['B1.txt']";
"B2 ['B2.txt']";
"C3 ['C3.txt']";
"B3 ['B3.txt']" -> "B2 ['B2.txt']";
"C4 ['C4.txt']" -> "C3 ['C3.txt']";
"C4 ['C4.txt']" -> "C2 ['C2.txt']";
"C2 ['C2.txt']" -> "C1 ['C1.txt']";
"C1 ['C1.txt']" -> "B2 ['B2.txt']";
"B2 ['B2.txt']" -> "B1 ['B1.txt']";
"C3 ['C3.txt']" -> "C1 ['C1.txt']";
}
''')
Base_Executor(wf).run(targets=['B1.txt'])
for f in ['A1.txt', 'A2.txt']:
self.assertFalse(FileTarget(f).exists())
for f in [
'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt',
'C4.txt'
]:
t = FileTarget(f)
self.assertTrue(t.exists())
t.remove('both')
#
# test 2, we would like to generate two files
dag = Base_Executor(wf).initialize_dag(targets=['B2.txt', 'C2.txt'])
# note that A2 is no longer mentioned
self.assertDAG(
dag, '''
strict digraph "" {
"C4 ['C4.txt']";
"B2 ['B2.txt']";
"C3 ['C3.txt']";
"B3 ['B3.txt']";
"C2 ['C2.txt']";
"C1 ['C1.txt']";
"C4 ['C4.txt']" -> "C2 ['C2.txt']";
"C4 ['C4.txt']" -> "C3 ['C3.txt']";
"C3 ['C3.txt']" -> "C1 ['C1.txt']";
"B3 ['B3.txt']" -> "B2 ['B2.txt']";
"C2 ['C2.txt']" -> "C1 ['C1.txt']";
"C1 ['C1.txt']" -> "B2 ['B2.txt']";
}
''')
Base_Executor(wf).run(targets=['B2.txt', 'C2.txt'])
for f in ['A1.txt', 'B1.txt', 'A2.txt']:
self.assertFalse(FileTarget(f).exists())
for f in ['C2.txt', 'B2.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']:
t = FileTarget(f)
self.assertTrue(t.exists())
t.remove('both')
#
# test 3, generate two separate trees
#
dag = Base_Executor(wf).initialize_dag(targets=['B3.txt', 'C2.txt'])
# note that A2 is no longer mentioned
self.assertDAG(
dag, '''
strict digraph "" {
"B3 ['B3.txt']";
"C2 ['C2.txt']";
"C4 ['C4.txt']";
"C4 ['C4.txt']" -> "C2 ['C2.txt']";
}
''')
Base_Executor(wf).run(targets=['B3.txt', 'C2.txt'])
for f in ['A1.txt', 'B1.txt', 'A2.txt', 'B2.txt', 'C1.txt', 'C3.txt']:
self.assertFalse(FileTarget(f).exists())
for f in ['C2.txt', 'B3.txt', 'C4.txt']:
t = FileTarget(f)
self.assertTrue(t.exists())
t.remove('both')
def testSimpleDAG(self):
'''Test DAG with simple dependency'''
for filename in ('a.txt', 'a1.txt'):
with open(filename, 'w') as tmp:
tmp.write('hey')
# basica case
# 1 -> 2 -> 3 -> 4
script = SoS_Script('''
[A_1]
[A_2]
[A_3]
[A_4]
''')
wf = script.workflow()
dag = Base_Executor(wf).initialize_dag()
self.assertDAG(
dag, '''strict digraph "" {
A_2;
A_4;
A_1;
A_3;
A_2 -> A_3;
A_1 -> A_2;
A_3 -> A_4;
}
''')
# basica case
# 1 -> 2 -> 3 -> 4
script = SoS_Script('''
[A_1]
[A_2]
[A_3]
input: 'a.txt'
[A_4]
''')
wf = script.workflow()
dag = Base_Executor(wf).initialize_dag()
self.assertDAG(
dag, '''strict digraph "" {
A_2;
A_4;
A_1;
A_3;
A_1 -> A_2;
A_3 -> A_4;
}
''')
#
# 1 -> 2 -> 3 -> 4
#
script = SoS_Script('''
[A_1]
input: 'a.txt'
output: 'b.txt'
[A_2]
input: 'b.txt'
output: 'c.txt'
[A_3]
input: 'c.txt'
output: 'd.txt'
[A_4]
input: 'd.txt'
output: 'e.txt'
''')
wf = script.workflow()
dag = Base_Executor(wf).initialize_dag()
self.assertDAG(
dag, '''strict digraph "" {
A_2;
A_4;
A_1;
A_3;
A_2 -> A_3;
A_1 -> A_2;
A_3 -> A_4;
}
''')
#
# 1 -> 2
# 3 -> 4 (3 does not have any input)
#
script = SoS_Script('''
[B_1]
input: 'a.txt'
output: 'b.txt'
[B_2]
input: 'b.txt'
output: 'c.txt'
[B_3]
input: []
output: 'd.txt'
[B_4]
input: 'd.txt'
output: 'e.txt'
''')
wf = script.workflow()
dag = Base_Executor(wf).initialize_dag()
self.assertDAG(
dag, '''strict digraph "" {
B_2;
B_4;
B_1;
B_3;
B_1 -> B_2;
B_3 -> B_4;
}
''')
#
# 1 -> 2
# 3 -> 4 (3 depends on something else)
#
script = SoS_Script('''
[B_1]
input: 'a.txt'
output: 'b.txt'
[B_2]
input: 'b.txt'
output: 'c.txt'
[B_3]
input: 'a1.txt'
output: 'd.txt'
[B_4]
input: 'd.txt'
output: 'e.txt'
''')
wf = script.workflow()
dag = Base_Executor(wf).initialize_dag()
self.assertDAG(
dag, '''strict digraph "" {
B_1;
B_4;
B_2;
B_3;
B_1 -> B_2;
B_3 -> B_4;
}
''')
#
# (1) -> 2
# (1) -> 3 -> 4
#
# 2 and 3 depends on the output of 1
script = SoS_Script('''
[C_1]
input: 'a.txt'
output: 'b.txt'
[C_2]
input: 'b.txt'
output: 'c.txt'
[C_3]
input: 'b.txt'
output: 'd.txt'
[C_4]
depends: 'd.txt'
output: 'e.txt'
''')
wf = script.workflow()
dag = Base_Executor(wf).initialize_dag()
self.assertDAG(
dag, '''
strict digraph "" {
C_1;
C_4;
C_2;
C_3;
C_1 -> C_2;
C_1 -> C_3;
C_3 -> C_4;
}
''')
for filename in ('a.txt', 'a1.txt'):
os.remove(filename)
def testPatternReuse(self):
'''Test repeated use of steps that use pattern and produce different files.'''
#
for f in [
'A1.txt', 'A2.txt', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p'
]:
FileTarget(f).remove('both')
#
# A1 <- P <- B1
# A1 <- P <- B2
# A2
#
script = SoS_Script('''
[A_1]
input: 'B1.txt.p', 'B2.txt.p'
output: 'A1.txt'
sh:
touch A1.txt
[A_2]
sh:
touch A2.txt
[B1: provides='B1.txt']
sh:
touch B1.txt
[B2: provides='B2.txt']
sh:
touch B2.txt
[P: provides='{filename}.p']
input: filename
sh:
touch ${output}
''')
# the workflow should call step K for step C_2, but not C_3
wf = script.workflow()
dag = Base_Executor(wf).initialize_dag()
self.assertDAG(
dag, '''
strict digraph "" {
"P ['B2.txt.p']";
"B1 ['B1.txt']";
"B2 ['B2.txt']";
A_2;
A_1;
"P ['B1.txt.p']";
"P ['B2.txt.p']" -> A_1;
"B1 ['B1.txt']" -> "P ['B1.txt.p']";
"B2 ['B2.txt']" -> "P ['B2.txt.p']";
A_1 -> A_2;
"P ['B1.txt.p']" -> A_1;
}
''')
Base_Executor(wf).run()
for f in [
'A1.txt', 'A2.txt', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p'
]:
t = FileTarget(f)
self.assertTrue(t.exists())
t.remove('both')
