def test_consume_full_sample_pipeline():
parser = Parser(full_sample_pipeline)
res = parser.consume()
res[0].alias.should.equal('1')
res[0].parts[0].should.equal(CommentToken(['#', ' Trimmomatic']))
res[0].parts[1].should.equal('java')
def test_one_step_pipeline():
parser = Parser(one_step_pipeline)
cmds = parser.consume()
vals = ['cut', 'somefile', '>', 'anotherfile']
for i, part in enumerate(cmds[0].parts):
vals[i].should.equal(part)
def test_one_step_pipeline():
parser = Parser(one_step_pipeline)
cmds = parser.consume()
vals = ["cut", "somefile", ">", "anotherfile"]
for i, part in enumerate(cmds[0].parts):
vals[i].should.equal(part)
def test_run_11():
parser = Parser(overall)
cmds = parser.consume()[:11]
pipeline = Runtime(cmds, ReportingJobQueue, JOB_TYPES, 'mock', sleep_time=0.01)
iters = pipeline.run()
iters.should.be.greater_than(15)
def test_get_new_commands_9():
parser = Parser(overall)
cmds = parser.consume()[:9]
pipeline = Runtime(cmds, ReportingJobQueue, JOB_TYPES, 'mock')
new = pipeline.queue.queue
new.should.have.length_of(9)
def test_max_concurrent_jobs():
parser = Parser(concurrent)
cmds = parser.consume()
pipeline = Runtime(cmds, ReportingJobQueue, { 'local': MockJob }, 'local', sleep_time=0.01)
iters = pipeline.run()
iters.should.be.greater_than(30)
def test_consume_full_sample_pipeline():
parser = Parser(full_sample_pipeline)
res = parser.consume()
res[0].alias.should.equal('1')
res[0].parts[0].should.equal(CommentToken(['#', ' Trimmomatic']))
res[0].parts[1].should.equal('java')
def test_full_sample_pipeline():
parser = Parser(full_sample_pipeline)
cmds = parser.consume()
vals = [
CommentToken(["#", " Trimmomatic"]),
"java",
"-jar",
PathToken("trimmomatic", "Trimmomatic-0.35/trimmomatic-0.35.jar>"),
"PE",
[[Input("*R1_001.fastq.gz")]],
[[Input("*R2_001.fastq.gz")]],
Output("1", "metapipe.1.output"),
Output("1", "metapipe.1.output"),
Output("1", "metapipe.1.output"),
Output("1", "metapipe.1.output"),
PathToken(
"illuminaclip",
"ILLUMINACLIP:/gpfs/home/bhuvan/Programs/Trimmomatic-0.32/adapters/TruSeq3-PE.fa:2:30:10:2:true",
),
"LEADING:3",
"TRAILING:3",
]
for i, part in enumerate(cmds[0].parts):
vals[i].should.equal(part)
def test_another_sample_pipeline():
parser = Parser(another_sample)
templates = parser.consume()
vals = [
CommentToken(["#", " Trimmomatic"]),
"java",
"-jar",
PathToken("trimmomatic", "Trimmomatic-0.35/trimmomatic-0.35.jar>"),
"PE",
Input("1"),
Input("2"),
Output("1.1-1", "metapipe.1.output"),
Output("1.1-2", "metapipe.1.output"),
Output("1.1-3", "metapipe.1.output"),
Output("1.1-4", "metapipe.1.output"),
"ILLUMINACLIP:Trimmomatic-0.35/adapters/TruSeq3-PE.fa:2:30:10:2:true",
"LEADING:3",
"TRAILING:3",
]
cmd = templates[0].eval()[0]
for i, part in enumerate(cmd.parts):
vals[i].should.equal(part)
def test_run_1():
parser = Parser(overall)
cmds = parser.consume()[:1]
pipeline = Runtime(cmds, JOB_TYPES, 'mock', sleep_time=0.01)
iters = pipeline.run()
iters.should.equal(1)
def test_run_10():
parser = Parser(overall)
cmds = parser.consume()[:10]
pipeline = Runtime(cmds, ReportingJobQueue, JOB_TYPES, 'mock', sleep_time=0.01)
iters = pipeline.run()
iters.should.equal(23)
def test_get_new_commands_9():
parser = Parser(overall)
cmds = parser.consume()[:9]
pipeline = Runtime(cmds, ReportingJobQueue, JOB_TYPES, 'mock')
new = pipeline.queue.queue
new.should.have.length_of(9)
def test_consume_files():
parser = Parser(overall)
res = parser.consume()
parser.files[0].alias.should.equal('1')
parser.files[0].filename.should.equal('somefile.1')
parser.files[1].alias.should.equal('2')
parser.files[1].filename.should.equal('somefile.2')
def test_consume_files():
parser = Parser(overall)
res = parser.consume()
parser.files[0].alias.should.equal('1')
parser.files[0].filename.should.equal('somefile.1')
parser.files[1].alias.should.equal('2')
parser.files[1].filename.should.equal('somefile.2')
def test_get_new_commands_8():
parser = Parser(overall)
cmds = parser.consume()[:8]
pipeline = Runtime(cmds, JOB_TYPES, 'mock')
new = pipeline._get_new_commands()
print(new)
new.should.have.length_of(11)
def test_consume_commands_9():
parser = Parser(overall)
res = parser.consume()
res[8].alias.should.equal('9')
res[8].parts[4].should.equal('cat')
res[8].parts[5][0][0].should.equal(Input('*.bam',
filename='*.bam'))
res[8]._dependencies.should.have.length_of(1)
def test_magical_glob():
parser = Parser(magical_glob2)
templates = parser.consume()
vals = ['cat', Input('1.*', ''), '>', Output('2.1', 'mp.2.1.output')]
cmd = templates[1].eval()[0]
for i, part in enumerate(cmd.parts):
vals[i].should.equal(part)
def test_multiple_outputs1():
parser = Parser(multiple_outputs)
cmds = parser.consume()
old_commands = []
cmd = cmds[0].eval()[0]
cmd.update_dependent_files(old_commands)
cmd.eval().should.equal('bash somescript somefile.1 --log'
' mp.1.1-1.output -r mp.1.1-2.output')
def test_eval_magic_input():
parser = Parser(magic_inputs)
templates = parser.consume()
vals = ["bash", "somescript", Input("*.counts", "somefile.1"), ">", Output("1.1", "metapipe.1.1.output")]
cmd = templates[0].eval()[0]
for i, part in enumerate(cmd.parts):
vals[i].should.equal(part)
def test_magical_glob():
parser = Parser(magical_glob2)
templates = parser.consume()
vals = ["cat", Input("1.*", ""), ">", Output("2.1", "mp.2.1.output")]
cmd = templates[1].eval()[0]
for i, part in enumerate(cmd.parts):
vals[i].should.equal(part)
def test_long_running_2():
parser = Parser(long_running)
cmds = parser.consume()
vals = ["cat", [[Input("1.1")], [Input("1.2")]], "&&", "sleep", "1"]
for i, part in enumerate(cmds[1].parts):
vals[i].should.equal(part)
def test_eval_2():
parser = Parser(overall)
cmds = parser.consume()
cmds[0].eval()[1].eval().should.equal('#PBS_O_WORKDIR=~/someuser\nset -e;'
'\nmodule load python\n# do something\n'
'/usr/bin/python somescript.py -i '
'somefile.4 somefile.5 somefile.6 -o mp.1.2.output '
'-fgh somefile.txt')
def test_consume_commands_9():
parser = Parser(overall)
res = parser.consume()
res[8].alias.should.equal('9')
res[8].parts[4].should.equal('cat')
res[8].parts[5][0][0].should.equal(Input('*.bam',
filename='*.bam'))
res[8]._dependencies.should.have.length_of(1)
def test_long_running_1():
parser = Parser(long_running)
old_commands = []
templates = parser.consume()
cmd = templates[0].eval()[0]
cmd.update_dependent_files(old_commands)
cmd.eval().should.equal('cat somefile.1 > mp.1.1.output && sleep 1')
def test_multiple_outputs2():
parser = Parser(multiple_outputs)
cmds = parser.consume()
old_commands = []
cmd = cmds[1].eval()[0]
cmd.update_dependent_files(old_commands)
cmd.eval().should.equal('python somescript.py somefile.4 somefile.5 '
'somefile.6 --log mp.2.1-1.output -r mp.2.1-2.output '
'--output mp.2.1-3.output')
def test_consume_commands_5():
parser = Parser(overall)
res = parser.consume()
res[4].alias.should.equal('5')
res[4].parts[4].should.equal('paste')
res[4].parts[5].should.equal('*.counts')
res[4].parts[6].should.equal('>')
res[4].parts[9].should.equal(Output('', magic='some.file'))
res[4]._dependencies.should.have.length_of(0)
def test_eval_multiple_inputs():
parser = Parser(multiple_inputs)
cmds = parser.consume()
old_commands = []
cmd = cmds[0].eval()[0]
print(cmd)
cmd.update_dependent_files(old_commands)
cmd.eval().should.equal('bash somescript somefile.1 --conf somefile.4 > '
'mp.1.1.output')
def test_eval_16_deps():
parser = Parser(overall)
cmds = parser.consume()
old_commands = []
for cmd in cmds[0:8]:
old_commands.extend(cmd.eval())
cmd = cmds[8].eval()[0]
cmd.update_dependent_files(old_commands)
cmd.depends_on.should.have.length_of(1)
def test_long_running_2():
parser = Parser(long_running)
templates = parser.consume()
vals = ["cat", Input("1.1", "metapipe.1.1.output"), "&&", "sleep", "1"]
cmd = templates[1].eval()[0]
for i, part in enumerate(cmd.parts):
vals[i].should.equal(part)
def test_output_file_name():
parser = Parser(full_output_file_name)
templates = parser.consume()
vals = ["gzip", "--stdout", Input("1", "somefile.1"), ">", Output("1.1", "metapipe.1.1.output.gz")]
cmd = templates[0].eval()[0]
for i, part in enumerate(cmd.parts):
vals[i].should.equal(part)
def test_consume_paths():
parser = Parser(overall)
res = parser.consume()
parser.paths[0].alias.should.equal('python')
parser.paths[0].path.should.equal('/usr/bin/python')
parser.paths[1].alias.should.equal('bash')
parser.paths[1].path.should.equal('/usr/bin/bash')
parser.paths[2].alias.should.equal('rb')
parser.paths[2].path.should.equal('/usr/bin/ruby')
def test_eval_7():
parser = Parser(overall)
cmds = parser.consume()
old_commands = []
for cmd in cmds[0:2]:
old_commands.extend(cmd.eval())
cmd = cmds[2].eval()[2]
cmd.update_dependent_files(old_commands)
cmd.eval().should.equal('/usr/bin/ruby somescript.rb -i metapipe.1.1.output metapipe.1.2.output >> somefile')
def test_long_running_2():
parser = Parser(long_running)
cmds = parser.consume()
vals = ['cat', [[Input('1.1')], [Input('1.2')]], '&&', 'sleep', '1']
for i, part in enumerate(cmds[1].parts):
vals[i].should.equal(part)
def test_eval_3():
parser = Parser(overall)
cmds = parser.consume()
old_commands = []
for cmd in cmds[0:1]:
old_commands.extend(cmd.eval())
cmd = cmds[1].eval()[0]
cmd.update_dependent_files(old_commands)
cmd.eval().should.equal('/usr/bin/bash somescript.sh -i metapipe.1.1.output -o metapipe.2.1.output -fgh somefile.txt')
def test_long_running_2():
parser = Parser(long_running)
templates = parser.consume()
vals = ['cat', Input('1.1', 'metapipe.1.1.output'), '&&', 'sleep', '1']
cmd = templates[1].eval()[0]
for i, part in enumerate(cmd.parts):
vals[i].should.equal(part)
def test_consume_paths():
parser = Parser(overall)
res = parser.consume()
parser.paths[0].alias.should.equal('python')
parser.paths[0].path.should.equal('/usr/bin/python')
parser.paths[1].alias.should.equal('bash')
parser.paths[1].path.should.equal('/usr/bin/bash')
parser.paths[2].alias.should.equal('rb')
parser.paths[2].path.should.equal('/usr/bin/ruby')
def test_max_concurrent_jobs():
parser = Parser(concurrent)
cmds = parser.consume()
pipeline = Runtime(cmds,
ReportingJobQueue, {'local': MockJob},
'local',
sleep_time=0.01)
iters = pipeline.run()
iters.should.be.greater_than(30)
def test_consume_commands_5():
parser = Parser(overall)
res = parser.consume()
res[4].alias.should.equal('5')
res[4].parts[4].should.equal('paste')
res[4].parts[5].should.equal('*.counts')
res[4].parts[6].should.equal('>')
res[4].parts[9].should.equal(Output('', magic='some.file'))
res[4]._dependencies.should.have.length_of(0)
def test_full_output_file_name():
parser = Parser(full_output_file_name)
templates = parser.consume()
old_commands = []
cmd = templates[0].eval()[0]
cmd.update_dependent_files(old_commands)
cmd.eval().should.equal('gzip --stdout somefile.1 > mp.1.1.output.gz')
def test_eval_6():
parser = Parser(overall)
cmds = parser.consume()
old_commands = []
for cmd in cmds[0:2]:
old_commands.extend(cmd.eval())
cmd = cmds[2].eval()[1]
cmd.update_dependent_files(old_commands)
print([i.filename for i in cmd.input_parts])
cmd.eval().should.equal('/usr/bin/ruby somescript.rb -i metapipe.2.2.output >> somefile')
def test_eval_16():
parser = Parser(overall)
cmds = parser.consume()
old_commands = []
for cmd in cmds[0:8]:
old_commands.extend(cmd.eval())
cmd = cmds[8].eval()[0]
cmd.update_dependent_files(old_commands)
print([i.eval() for i in cmd.input_parts])
cmd.eval().should.equal('cat somefile.1.bam somefile.2.bam somefile.bam')
def test_run_10():
parser = Parser(overall)
cmds = parser.consume()[:10]
pipeline = Runtime(cmds,
ReportingJobQueue,
JOB_TYPES,
'mock',
sleep_time=0.01)
iters = pipeline.run()
iters.should.equal(25)
def test_eval_9():
parser = Parser(overall)
cmds = parser.consume()
old_commands = []
for cmd in cmds[0:4]:
old_commands.extend(cmd.eval())
cmd = cmds[4].eval()[0]
cmd.update_dependent_files(old_commands)
print([i.filename for i in cmd.input_parts])
cmd.eval().should.equal('paste *.counts > some.file # some.file')
def test_consume_commands_4():
parser = Parser(overall)
res = parser.consume()
res[3].alias.should.equal('4')
res[3].parts[4].should.equal('cut')
res[3].parts[5].should.equal('-f')
res[3].parts[6].should.equal('*.counts')
res[3].parts[7][0].should.equal('>')
res[3].parts[8].should.equal('something.file')
res[3]._dependencies.should.have.length_of(0)
def test_run_11():
parser = Parser(overall)
cmds = parser.consume()[:11]
pipeline = Runtime(cmds,
ReportingJobQueue,
JOB_TYPES,
'mock',
sleep_time=0.01)
iters = pipeline.run()
iters.should.be.greater_than(15)
def test_consume_commands_6():
parser = Parser(overall)
res = parser.consume()
print(res[5].parts)
res[5].alias.should.equal('6')
res[5].parts[4].should.equal('./somescript')
res[5].parts[5][0][0].should.equal(Input('1', 'somefile.1'))
res[5].parts[5][0][1].should.equal(Input('2', 'somefile.2'))
res[5].parts[5][0][2].should.equal(Input('3', 'somefile.3'))
res[5].parts[5][1][0].should.equal(Input('4', '*.counts'))
res[5]._dependencies.should.have.length_of(0)
def test_eval_9():
parser = Parser(overall)
cmds = parser.consume()
old_commands = []
for cmd in cmds[0:4]:
old_commands.extend(cmd.eval())
cmd = cmds[4].eval()[0]
cmd.update_dependent_files(old_commands)
cmd.eval().should.equal('#PBS_O_WORKDIR=~/someuser\nset -e;'
'\nmodule load python\n# do something\n'
'paste *.counts > some.file # some.file')
def test_eval_16():
parser = Parser(overall)
cmds = parser.consume()
old_commands = []
for cmd in cmds[0:8]:
old_commands.extend(cmd.eval())
cmd = cmds[8].eval()[0]
cmd.update_dependent_files(old_commands)
cmd.eval().should.equal('#PBS_O_WORKDIR=~/someuser\nset -e;'
'\nmodule load python\n# do something\n'
'cat somefile.1.bam somefile.2.bam somefile.bam')
def test_eval_14():
parser = Parser(overall)
cmds = parser.consume()
old_commands = []
for cmd in cmds[0:6]:
old_commands.extend(cmd.eval())
cmd = cmds[6].eval()[3]
cmd.update_dependent_files(old_commands)
cmd.eval().should.equal('#PBS_O_WORKDIR=~/someuser\nset -e;'
'\nmodule load python\n# do something\n'
'/usr/bin/ruby somescript.rb -i somefile.4.counts')
def test_consume_commands_7():
parser = Parser(overall)
res = parser.consume()
res[6].alias.should.equal('7')
res[6].parts[4].should.equal(parser.paths[2])
res[6].parts[5].should.equal('somescript.rb')
res[6].parts[6].should.equal('-i')
res[6].parts[7][0][0].should.equal(Input('*.counts',
'*.counts'))
res[6].parts.should.have.length_of(8)
res[6]._dependencies.should.have.length_of(0)
def test_consume_commands_8():
parser = Parser(overall)
res = parser.consume()
res[7].alias.should.equal('8')
res[7].parts[4].should.equal(parser.paths[0])
res[7].parts[5].should.equal('somescript.py')
res[7].parts[6].should.equal('-i')
res[7].parts[7][0][0].should.equal(Input('*.counts',
filename='*.counts'))
res[7].parts[9].should.equal(Output('', magic='*.bam'))
res[7]._dependencies.should.have.length_of(0)
def test_eval_7():
parser = Parser(overall)
cmds = parser.consume()
old_commands = []
for cmd in cmds[0:2]:
old_commands.extend(cmd.eval())
cmd = cmds[2].eval()[2]
cmd.update_dependent_files(old_commands)
cmd.eval().should.equal('#PBS_O_WORKDIR=~/someuser\nset -e;'
'\nmodule load python\n# do something\n/usr/bin/ruby somescript.rb -i '
'mp.1.1.output mp.1.2.output >> somefile')
def test_full_output_file_name_2():
parser = Parser(full_output_file_name)
templates = parser.consume()
old_commands = []
for cmd in templates[0:1]:
old_commands.extend(cmd.eval())
cmd = templates[1].eval()[0]
cmd.update_dependent_files(old_commands)
cmd.eval().should.equal('cat mp.1.1.output.gz > mp.2.1.output.gz')
def test_eval_4():
parser = Parser(overall)
cmds = parser.consume()
old_commands = []
for cmd in cmds[0:1]:
old_commands.extend(cmd.eval())
cmd = cmds[1].eval()[1]
cmd.update_dependent_files(old_commands)
cmd.eval().should.equal('#PBS_O_WORKDIR=~/someuser\nset -e;'
'\nmodule load python\n# do something\n'
'/usr/bin/bash somescript.sh -i mp.1.2.output'
' -o mp.2.2.output -fgh somefile.txt')