Esempio n. 1
0
def test_consume_full_sample_pipeline():
    parser = Parser(full_sample_pipeline)
    res = parser.consume()

    res[0].alias.should.equal('1')
    res[0].parts[0].should.equal(CommentToken(['#', ' Trimmomatic']))
    res[0].parts[1].should.equal('java')
def test_one_step_pipeline():
    parser = Parser(one_step_pipeline)
    cmds = parser.consume()

    vals = ['cut', 'somefile', '>', 'anotherfile']
    for i, part in enumerate(cmds[0].parts):
        vals[i].should.equal(part)
def test_one_step_pipeline():
    parser = Parser(one_step_pipeline)
    cmds = parser.consume()

    vals = ["cut", "somefile", ">", "anotherfile"]
    for i, part in enumerate(cmds[0].parts):
        vals[i].should.equal(part)
Esempio n. 4
0
def test_run_11():
    parser = Parser(overall)
    cmds = parser.consume()[:11]

    pipeline = Runtime(cmds, ReportingJobQueue, JOB_TYPES, 'mock', sleep_time=0.01)
    iters = pipeline.run()
    iters.should.be.greater_than(15)
Esempio n. 5
0
def test_get_new_commands_9():
    parser = Parser(overall)
    cmds = parser.consume()[:9]

    pipeline = Runtime(cmds, ReportingJobQueue, JOB_TYPES, 'mock')
    new = pipeline.queue.queue
    new.should.have.length_of(9)
Esempio n. 6
0
def test_max_concurrent_jobs():
    parser = Parser(concurrent)
    cmds = parser.consume()

    pipeline = Runtime(cmds, ReportingJobQueue, { 'local': MockJob }, 'local', sleep_time=0.01)
    iters = pipeline.run()
    iters.should.be.greater_than(30)
Esempio n. 7
0
def test_consume_full_sample_pipeline():
    parser = Parser(full_sample_pipeline)
    res = parser.consume()

    res[0].alias.should.equal('1')
    res[0].parts[0].should.equal(CommentToken(['#', ' Trimmomatic']))
    res[0].parts[1].should.equal('java')
def test_full_sample_pipeline():
    parser = Parser(full_sample_pipeline)

    cmds = parser.consume()

    vals = [
        CommentToken(["#", " Trimmomatic"]),
        "java",
        "-jar",
        PathToken("trimmomatic", "Trimmomatic-0.35/trimmomatic-0.35.jar>"),
        "PE",
        [[Input("*R1_001.fastq.gz")]],
        [[Input("*R2_001.fastq.gz")]],
        Output("1", "metapipe.1.output"),
        Output("1", "metapipe.1.output"),
        Output("1", "metapipe.1.output"),
        Output("1", "metapipe.1.output"),
        PathToken(
            "illuminaclip",
            "ILLUMINACLIP:/gpfs/home/bhuvan/Programs/Trimmomatic-0.32/adapters/TruSeq3-PE.fa:2:30:10:2:true",
        ),
        "LEADING:3",
        "TRAILING:3",
    ]

    for i, part in enumerate(cmds[0].parts):
        vals[i].should.equal(part)
def test_another_sample_pipeline():
    parser = Parser(another_sample)

    templates = parser.consume()

    vals = [
        CommentToken(["#", " Trimmomatic"]),
        "java",
        "-jar",
        PathToken("trimmomatic", "Trimmomatic-0.35/trimmomatic-0.35.jar>"),
        "PE",
        Input("1"),
        Input("2"),
        Output("1.1-1", "metapipe.1.output"),
        Output("1.1-2", "metapipe.1.output"),
        Output("1.1-3", "metapipe.1.output"),
        Output("1.1-4", "metapipe.1.output"),
        "ILLUMINACLIP:Trimmomatic-0.35/adapters/TruSeq3-PE.fa:2:30:10:2:true",
        "LEADING:3",
        "TRAILING:3",
    ]

    cmd = templates[0].eval()[0]
    for i, part in enumerate(cmd.parts):
        vals[i].should.equal(part)
Esempio n. 10
0
def test_run_1():
    parser = Parser(overall)
    cmds = parser.consume()[:1]
    
    pipeline = Runtime(cmds, JOB_TYPES, 'mock', sleep_time=0.01)
    iters = pipeline.run()
    iters.should.equal(1)
Esempio n. 11
0
def test_run_10():
    parser = Parser(overall)
    cmds = parser.consume()[:10]

    pipeline = Runtime(cmds, ReportingJobQueue, JOB_TYPES, 'mock', sleep_time=0.01)
    iters = pipeline.run()
    iters.should.equal(23)
Esempio n. 12
0
def test_get_new_commands_9():
    parser = Parser(overall)
    cmds = parser.consume()[:9]

    pipeline = Runtime(cmds, ReportingJobQueue, JOB_TYPES, 'mock')
    new = pipeline.queue.queue
    new.should.have.length_of(9)
Esempio n. 13
0
def test_consume_files():
    parser = Parser(overall)
    res = parser.consume()

    parser.files[0].alias.should.equal('1')
    parser.files[0].filename.should.equal('somefile.1')
    parser.files[1].alias.should.equal('2')
    parser.files[1].filename.should.equal('somefile.2')
Esempio n. 14
0
def test_consume_files():
    parser = Parser(overall)
    res = parser.consume()

    parser.files[0].alias.should.equal('1')
    parser.files[0].filename.should.equal('somefile.1')
    parser.files[1].alias.should.equal('2')
    parser.files[1].filename.should.equal('somefile.2')
Esempio n. 15
0
def test_get_new_commands_8():
    parser = Parser(overall)
    cmds = parser.consume()[:8]

    pipeline = Runtime(cmds, JOB_TYPES, 'mock')
    new = pipeline._get_new_commands()
    print(new)
    new.should.have.length_of(11)    
Esempio n. 16
0
def test_consume_commands_9():
    parser = Parser(overall)
    res = parser.consume()

    res[8].alias.should.equal('9')
    res[8].parts[4].should.equal('cat')
    res[8].parts[5][0][0].should.equal(Input('*.bam',
                                filename='*.bam'))
    res[8]._dependencies.should.have.length_of(1)
Esempio n. 17
0
def test_magical_glob():
    parser = Parser(magical_glob2)
    templates = parser.consume()

    vals = ['cat', Input('1.*', ''), '>', Output('2.1', 'mp.2.1.output')]

    cmd = templates[1].eval()[0]
    for i, part in enumerate(cmd.parts):
        vals[i].should.equal(part)
Esempio n. 18
0
def test_multiple_outputs1():
    parser = Parser(multiple_outputs)
    cmds = parser.consume()
    old_commands = []

    cmd = cmds[0].eval()[0]
    cmd.update_dependent_files(old_commands)
    cmd.eval().should.equal('bash somescript somefile.1 --log'
        ' mp.1.1-1.output -r mp.1.1-2.output')
def test_eval_magic_input():
    parser = Parser(magic_inputs)

    templates = parser.consume()

    vals = ["bash", "somescript", Input("*.counts", "somefile.1"), ">", Output("1.1", "metapipe.1.1.output")]
    cmd = templates[0].eval()[0]
    for i, part in enumerate(cmd.parts):
        vals[i].should.equal(part)
def test_magical_glob():
    parser = Parser(magical_glob2)
    templates = parser.consume()

    vals = ["cat", Input("1.*", ""), ">", Output("2.1", "mp.2.1.output")]

    cmd = templates[1].eval()[0]
    for i, part in enumerate(cmd.parts):
        vals[i].should.equal(part)
def test_long_running_2():
    parser = Parser(long_running)

    cmds = parser.consume()

    vals = ["cat", [[Input("1.1")], [Input("1.2")]], "&&", "sleep", "1"]

    for i, part in enumerate(cmds[1].parts):
        vals[i].should.equal(part)
Esempio n. 22
0
def test_eval_2():
    parser = Parser(overall)
    cmds = parser.consume()

    cmds[0].eval()[1].eval().should.equal('#PBS_O_WORKDIR=~/someuser\nset -e;'
        '\nmodule load python\n# do something\n'
        '/usr/bin/python somescript.py -i '
        'somefile.4 somefile.5 somefile.6 -o mp.1.2.output '
        '-fgh somefile.txt')
Esempio n. 23
0
def test_consume_commands_9():
    parser = Parser(overall)
    res = parser.consume()

    res[8].alias.should.equal('9')
    res[8].parts[4].should.equal('cat')
    res[8].parts[5][0][0].should.equal(Input('*.bam',
                                filename='*.bam'))
    res[8]._dependencies.should.have.length_of(1)
Esempio n. 24
0
def test_long_running_1():
    parser = Parser(long_running)

    old_commands = []

    templates = parser.consume()

    cmd = templates[0].eval()[0]
    cmd.update_dependent_files(old_commands)
    cmd.eval().should.equal('cat somefile.1 > mp.1.1.output && sleep 1')
Esempio n. 25
0
def test_multiple_outputs2():
    parser = Parser(multiple_outputs)
    cmds = parser.consume()
    old_commands = []

    cmd = cmds[1].eval()[0]
    cmd.update_dependent_files(old_commands)
    cmd.eval().should.equal('python somescript.py somefile.4 somefile.5 '
        'somefile.6 --log mp.2.1-1.output -r mp.2.1-2.output '
        '--output mp.2.1-3.output')
Esempio n. 26
0
def test_consume_commands_5():
    parser = Parser(overall)
    res = parser.consume()

    res[4].alias.should.equal('5')
    res[4].parts[4].should.equal('paste')
    res[4].parts[5].should.equal('*.counts')
    res[4].parts[6].should.equal('>')
    res[4].parts[9].should.equal(Output('', magic='some.file'))
    res[4]._dependencies.should.have.length_of(0)
Esempio n. 27
0
def test_eval_multiple_inputs():
    parser = Parser(multiple_inputs)
    cmds = parser.consume()
    old_commands = []

    cmd = cmds[0].eval()[0]
    print(cmd)
    cmd.update_dependent_files(old_commands)
    cmd.eval().should.equal('bash somescript somefile.1 --conf somefile.4 > '
        'mp.1.1.output')
Esempio n. 28
0
def test_eval_16_deps():
    parser = Parser(overall)
    cmds = parser.consume()
    old_commands = []
    for cmd in cmds[0:8]:
        old_commands.extend(cmd.eval())

    cmd = cmds[8].eval()[0]
    cmd.update_dependent_files(old_commands)
    cmd.depends_on.should.have.length_of(1)
def test_long_running_2():
    parser = Parser(long_running)

    templates = parser.consume()

    vals = ["cat", Input("1.1", "metapipe.1.1.output"), "&&", "sleep", "1"]

    cmd = templates[1].eval()[0]
    for i, part in enumerate(cmd.parts):
        vals[i].should.equal(part)
def test_output_file_name():
    parser = Parser(full_output_file_name)

    templates = parser.consume()

    vals = ["gzip", "--stdout", Input("1", "somefile.1"), ">", Output("1.1", "metapipe.1.1.output.gz")]

    cmd = templates[0].eval()[0]
    for i, part in enumerate(cmd.parts):
        vals[i].should.equal(part)
Esempio n. 31
0
def test_consume_paths():
    parser = Parser(overall)
    res = parser.consume()

    parser.paths[0].alias.should.equal('python')
    parser.paths[0].path.should.equal('/usr/bin/python')
    parser.paths[1].alias.should.equal('bash')
    parser.paths[1].path.should.equal('/usr/bin/bash')
    parser.paths[2].alias.should.equal('rb')
    parser.paths[2].path.should.equal('/usr/bin/ruby')
Esempio n. 32
0
def test_eval_7():
    parser = Parser(overall)
    cmds = parser.consume()
    old_commands = []
    for cmd in cmds[0:2]:
        old_commands.extend(cmd.eval())

    cmd = cmds[2].eval()[2]
    cmd.update_dependent_files(old_commands)
    cmd.eval().should.equal('/usr/bin/ruby somescript.rb -i metapipe.1.1.output metapipe.1.2.output >> somefile')
def test_long_running_2():
    parser = Parser(long_running)

    cmds = parser.consume()


    vals = ['cat', [[Input('1.1')], [Input('1.2')]], '&&', 'sleep', '1']

    for i, part in enumerate(cmds[1].parts):
        vals[i].should.equal(part)
Esempio n. 34
0
def test_eval_3():
    parser = Parser(overall)
    cmds = parser.consume()
    old_commands = []
    for cmd in cmds[0:1]:
        old_commands.extend(cmd.eval())

    cmd = cmds[1].eval()[0]
    cmd.update_dependent_files(old_commands)
    cmd.eval().should.equal('/usr/bin/bash somescript.sh -i metapipe.1.1.output -o metapipe.2.1.output -fgh somefile.txt')
Esempio n. 35
0
def test_long_running_2():
    parser = Parser(long_running)

    templates = parser.consume()

    vals = ['cat', Input('1.1', 'metapipe.1.1.output'), '&&', 'sleep', '1']

    cmd = templates[1].eval()[0]
    for i, part in enumerate(cmd.parts):
        vals[i].should.equal(part)
Esempio n. 36
0
def test_consume_paths():
    parser = Parser(overall)
    res = parser.consume()

    parser.paths[0].alias.should.equal('python')
    parser.paths[0].path.should.equal('/usr/bin/python')
    parser.paths[1].alias.should.equal('bash')
    parser.paths[1].path.should.equal('/usr/bin/bash')
    parser.paths[2].alias.should.equal('rb')
    parser.paths[2].path.should.equal('/usr/bin/ruby')
Esempio n. 37
0
def test_max_concurrent_jobs():
    parser = Parser(concurrent)
    cmds = parser.consume()

    pipeline = Runtime(cmds,
                       ReportingJobQueue, {'local': MockJob},
                       'local',
                       sleep_time=0.01)
    iters = pipeline.run()
    iters.should.be.greater_than(30)
Esempio n. 38
0
def test_consume_commands_5():
    parser = Parser(overall)
    res = parser.consume()

    res[4].alias.should.equal('5')
    res[4].parts[4].should.equal('paste')
    res[4].parts[5].should.equal('*.counts')
    res[4].parts[6].should.equal('>')
    res[4].parts[9].should.equal(Output('', magic='some.file'))
    res[4]._dependencies.should.have.length_of(0)
Esempio n. 39
0
def test_full_output_file_name():
    parser = Parser(full_output_file_name)

    templates = parser.consume()

    old_commands = []

    cmd = templates[0].eval()[0]

    cmd.update_dependent_files(old_commands)
    cmd.eval().should.equal('gzip --stdout somefile.1 > mp.1.1.output.gz')
Esempio n. 40
0
def test_eval_6():
    parser = Parser(overall)
    cmds = parser.consume()
    old_commands = []
    for cmd in cmds[0:2]:
        old_commands.extend(cmd.eval())

    cmd = cmds[2].eval()[1]
    cmd.update_dependent_files(old_commands)
    print([i.filename for i in cmd.input_parts])
    cmd.eval().should.equal('/usr/bin/ruby somescript.rb -i metapipe.2.2.output >> somefile')
Esempio n. 41
0
def test_eval_16():
    parser = Parser(overall)
    cmds = parser.consume()
    old_commands = []
    for cmd in cmds[0:8]:
        old_commands.extend(cmd.eval())

    cmd = cmds[8].eval()[0]
    cmd.update_dependent_files(old_commands)
    print([i.eval() for i in cmd.input_parts])
    cmd.eval().should.equal('cat somefile.1.bam somefile.2.bam somefile.bam')
Esempio n. 42
0
def test_run_10():
    parser = Parser(overall)
    cmds = parser.consume()[:10]

    pipeline = Runtime(cmds,
                       ReportingJobQueue,
                       JOB_TYPES,
                       'mock',
                       sleep_time=0.01)
    iters = pipeline.run()
    iters.should.equal(25)
Esempio n. 43
0
def test_eval_9():
    parser = Parser(overall)
    cmds = parser.consume()
    old_commands = []
    for cmd in cmds[0:4]:
        old_commands.extend(cmd.eval())

    cmd = cmds[4].eval()[0]
    cmd.update_dependent_files(old_commands)
    print([i.filename for i in cmd.input_parts])
    cmd.eval().should.equal('paste *.counts > some.file # some.file')
Esempio n. 44
0
def test_consume_commands_4():
    parser = Parser(overall)
    res = parser.consume()

    res[3].alias.should.equal('4')
    res[3].parts[4].should.equal('cut')
    res[3].parts[5].should.equal('-f')
    res[3].parts[6].should.equal('*.counts')
    res[3].parts[7][0].should.equal('>')
    res[3].parts[8].should.equal('something.file')
    res[3]._dependencies.should.have.length_of(0)
Esempio n. 45
0
def test_run_11():
    parser = Parser(overall)
    cmds = parser.consume()[:11]

    pipeline = Runtime(cmds,
                       ReportingJobQueue,
                       JOB_TYPES,
                       'mock',
                       sleep_time=0.01)
    iters = pipeline.run()
    iters.should.be.greater_than(15)
Esempio n. 46
0
def test_consume_commands_6():
    parser = Parser(overall)
    res = parser.consume()
    print(res[5].parts)
    res[5].alias.should.equal('6')
    res[5].parts[4].should.equal('./somescript')
    res[5].parts[5][0][0].should.equal(Input('1', 'somefile.1'))
    res[5].parts[5][0][1].should.equal(Input('2', 'somefile.2'))
    res[5].parts[5][0][2].should.equal(Input('3', 'somefile.3'))
    res[5].parts[5][1][0].should.equal(Input('4', '*.counts'))
    res[5]._dependencies.should.have.length_of(0)
Esempio n. 47
0
def test_eval_9():
    parser = Parser(overall)
    cmds = parser.consume()
    old_commands = []
    for cmd in cmds[0:4]:
        old_commands.extend(cmd.eval())

    cmd = cmds[4].eval()[0]
    cmd.update_dependent_files(old_commands)
    cmd.eval().should.equal('#PBS_O_WORKDIR=~/someuser\nset -e;'
        '\nmodule load python\n# do something\n'
        'paste *.counts > some.file # some.file')
Esempio n. 48
0
def test_eval_16():
    parser = Parser(overall)
    cmds = parser.consume()
    old_commands = []
    for cmd in cmds[0:8]:
        old_commands.extend(cmd.eval())

    cmd = cmds[8].eval()[0]
    cmd.update_dependent_files(old_commands)
    cmd.eval().should.equal('#PBS_O_WORKDIR=~/someuser\nset -e;'
        '\nmodule load python\n# do something\n'
        'cat somefile.1.bam somefile.2.bam somefile.bam')
Esempio n. 49
0
def test_eval_14():
    parser = Parser(overall)
    cmds = parser.consume()
    old_commands = []
    for cmd in cmds[0:6]:
        old_commands.extend(cmd.eval())

    cmd = cmds[6].eval()[3]
    cmd.update_dependent_files(old_commands)
    cmd.eval().should.equal('#PBS_O_WORKDIR=~/someuser\nset -e;'
        '\nmodule load python\n# do something\n'
        '/usr/bin/ruby somescript.rb -i somefile.4.counts')
Esempio n. 50
0
def test_consume_commands_7():
    parser = Parser(overall)
    res = parser.consume()

    res[6].alias.should.equal('7')
    res[6].parts[4].should.equal(parser.paths[2])
    res[6].parts[5].should.equal('somescript.rb')
    res[6].parts[6].should.equal('-i')
    res[6].parts[7][0][0].should.equal(Input('*.counts',
                                        '*.counts'))
    res[6].parts.should.have.length_of(8)
    res[6]._dependencies.should.have.length_of(0)
Esempio n. 51
0
def test_consume_commands_8():
    parser = Parser(overall)
    res = parser.consume()

    res[7].alias.should.equal('8')
    res[7].parts[4].should.equal(parser.paths[0])
    res[7].parts[5].should.equal('somescript.py')
    res[7].parts[6].should.equal('-i')
    res[7].parts[7][0][0].should.equal(Input('*.counts',
                                filename='*.counts'))
    res[7].parts[9].should.equal(Output('', magic='*.bam'))
    res[7]._dependencies.should.have.length_of(0)
Esempio n. 52
0
def test_eval_7():
    parser = Parser(overall)
    cmds = parser.consume()
    old_commands = []
    for cmd in cmds[0:2]:
        old_commands.extend(cmd.eval())

    cmd = cmds[2].eval()[2]
    cmd.update_dependent_files(old_commands)
    cmd.eval().should.equal('#PBS_O_WORKDIR=~/someuser\nset -e;'
        '\nmodule load python\n# do something\n/usr/bin/ruby somescript.rb -i '
        'mp.1.1.output mp.1.2.output >> somefile')
Esempio n. 53
0
def test_full_output_file_name_2():
    parser = Parser(full_output_file_name)

    templates = parser.consume()

    old_commands = []

    for cmd in templates[0:1]:
        old_commands.extend(cmd.eval())
    cmd = templates[1].eval()[0]

    cmd.update_dependent_files(old_commands)
    cmd.eval().should.equal('cat mp.1.1.output.gz > mp.2.1.output.gz')
Esempio n. 54
0
def test_eval_4():
    parser = Parser(overall)
    cmds = parser.consume()
    old_commands = []
    for cmd in cmds[0:1]:
        old_commands.extend(cmd.eval())

    cmd = cmds[1].eval()[1]
    cmd.update_dependent_files(old_commands)
    cmd.eval().should.equal('#PBS_O_WORKDIR=~/someuser\nset -e;'
        '\nmodule load python\n# do something\n'
        '/usr/bin/bash somescript.sh -i mp.1.2.output'
        ' -o mp.2.2.output -fgh somefile.txt')