def test_no_duplicated_jobs_after_file_query_direct(db, tmpdir):
if not db.startswith('mysql'):
db = os.path.join(str(tmpdir), db)
jip.db.init(db)
p = jip.Pipeline()
a = p.bash('ls ${input}', input='A.txt', output='out.dat')
p.context(locals())
jobs = jip.create_jobs(p, validate=False)
jip.db.save(jobs)
assert len(list(jip.db.get_all())) == 1
assert jobs[0].id == 1
# second pipeline
p = jip.Pipeline()
a = p.bash('ls ${input}', input='A.txt', output='out.dat')
b = p.bash('ls ${input}', input=a)
p.context(locals())
jobs = jip.create_jobs(p, validate=False)
# search for the out.dat job
existing = jip.db.query_by_files(
outputs=jobs[1].tool.input.value
)
assert len(list(existing)) == 1
old = list(existing)[0]
# now replace the dependency
jobs[1].dependencies = [old]
# save only job 1
jip.db.save(jobs[1])
# we should have 2 jobs in the database
assert len(list(jip.db.get_all())) == 2
# and the one we skipped has no ID
assert jobs[0].id is None
def test_bash_tool_job_rendering():
p = Pipeline()
p.run('bash', cmd="testme", output="test.out")
jobs = create_jobs(p)
assert len(jobs) == 1
assert jobs[0].command == "(testme)> " + os.path.join(
os.getcwd(), 'test.out')
def test_single_job_fail(tmpdir):
tmpdir = str(tmpdir)
target_file = os.path.join(tmpdir, 'result.txt')
db_file = os.path.join(tmpdir, "test.db")
assert not os.path.exists(target_file)
# create a JIP database and a session
jip.db.init(db_file)
# create the cluster instance
c = cl.LocalCluster()
# create the pipeline
p = jip.Pipeline()
p.job(dir=tmpdir).bash('touch ${input}; exit 1;', outfile=target_file)
p.context(locals())
# create the jobs
jobs = jip.create_jobs(p)
# iterate the executions and pass the session so all jobs are stored
for e in jip.create_executions(jobs, save=True):
jip.submit_job(e.job, cluster=c)
c.wait()
# now the file should be there
assert not os.path.exists(target_file)
# we should also have the log files
assert os.path.exists(os.path.join(tmpdir, "jip-1.out"))
assert os.path.exists(os.path.join(tmpdir, "jip-1.err"))
# and we should have one job in Done state in our database
# we do the query with a fresh session though
job = jip.db.get(1)
assert job is not None
assert job.state == jip.db.STATE_FAILED
def test_tool_name_in_pipeline_context():
@jip.tool()
class MyTool():
def validate(self):
self.job.name = "testtool"
def get_command(self):
return "echo"
@jip.pipeline()
class MyPipeline():
def validate(self):
self.name("thepipeline")
def pipeline(self):
p = jip.Pipeline()
p.run('MyTool')
return p
p = jip.Pipeline()
p.run('MyPipeline')
p.expand()
jobs = jip.create_jobs(p)
assert len(jobs) == 1
assert jobs[0].name == "testtool"
assert jobs[0].pipeline == "thepipeline"
def test_pipeline_tool_defaults_global_job():
@jip.tool()
class MyTool():
def setup(self):
self.profile.threads = 2
self.profile.queue = "Org"
def get_command(self):
return "echo"
@jip.pipeline()
class MyPipeline():
def pipeline(self):
p = jip.Pipeline()
p.job(threads=3, queue="Intern").run('MyTool')
return p
p = jip.Pipeline()
p.run('MyPipeline')
p.expand()
profile = jip.Profile(threads=5, queue="yeah")
profile.specs['MyTool'] = jip.Profile()
profile.apply_to_pipeline(p)
jobs = jip.create_jobs(p)
assert jobs[0].threads == 3
assert jobs[0].queue == "Intern"
def test_tool_name_in_pipelines_with_multiplexing_and_custom_template_name():
@jip.tool()
class MyTool():
"""mytool
usage:
mytool <data>
"""
def validate(self):
self.job.name = "${data}"
def get_command(self):
return "echo"
@jip.pipeline()
class MyPipeline():
def validate(self):
self.name("thepipeline")
def pipeline(self):
p = jip.Pipeline()
p.run('MyTool', data=["A", "B"])
return p
p = jip.Pipeline()
p.run('MyPipeline')
p.expand()
profile = jip.Profile(name="customname")
jobs = jip.create_jobs(p, profile=profile)
assert len(jobs) == 2
assert jobs[0].name == "A"
assert jobs[0].pipeline == "customname"
assert jobs[1].name == "B"
assert jobs[1].pipeline == "customname"
def test_tool_name_with_local_context():
p = jip.Pipeline()
a = p.run('foo', input='Makefile')
p.context(locals())
jobs = jip.create_jobs(p, validate=False)
assert len(jobs) == 1
assert jobs[0].name == 'Makefile'
def test_setting_working_directory_to_sub():
cwd = os.getcwd()
p = jip.Pipeline()
# produce n files
p.job(dir="sub").run('produce', prefix='test', number=5)
jobs = jip.create_jobs(p)
assert jobs[0].working_directory == cwd + "/sub"
def test_tool_name_in_pipelines_with_multiplexing_and_custom_template_name():
@jip.tool()
class MyTool:
"""mytool
usage:
mytool <data>
"""
def validate(self):
self.job.name = "${data}"
def get_command(self):
return "echo"
@jip.pipeline()
class MyPipeline:
def validate(self):
self.name("thepipeline")
def pipeline(self):
p = jip.Pipeline()
p.run("MyTool", data=["A", "B"])
return p
p = jip.Pipeline()
p.run("MyPipeline")
p.expand()
profile = jip.Profile(name="customname")
jobs = jip.create_jobs(p, profile=profile)
assert len(jobs) == 2
assert jobs[0].name == "A"
assert jobs[0].pipeline == "customname"
assert jobs[1].name == "B"
assert jobs[1].pipeline == "customname"
def test_tool_name_in_pipeline_context_with_custom_profile_and_custom_name():
@jip.tool()
class MyTool:
def validate(self):
self.job.name = "testtool"
def get_command(self):
return "echo"
@jip.pipeline()
class MyPipeline:
def validate(self):
self.name("thepipeline")
def pipeline(self):
p = jip.Pipeline()
p.job("Tool1").run("MyTool")
return p
p = jip.Pipeline()
p.run("MyPipeline")
p.expand()
profile = jip.Profile(name="customname")
jobs = jip.create_jobs(p, profile=profile)
assert len(jobs) == 1
assert jobs[0].name == "Tool1"
assert jobs[0].pipeline == "customname"
def test_single_job_fail(tmpdir):
tmpdir = str(tmpdir)
target_file = os.path.join(tmpdir, 'result.txt')
db_file = os.path.join(tmpdir, "test.db")
assert not os.path.exists(target_file)
# create a JIP database and a session
jip.db.init(db_file)
# create the cluster instance
c = cl.LocalCluster()
# create the pipeline
p = jip.Pipeline()
p.job(dir=tmpdir).bash('touch ${input}; exit 1;', outfile=target_file)
p.context(locals())
# create the jobs
jobs = jip.create_jobs(p)
# iterate the executions and pass the session so all jobs are stored
for e in jip.create_executions(jobs, save=True):
jip.submit_job(e.job, cluster=c)
c.wait()
# now the file should be there
assert not os.path.exists(target_file)
# we should also have the log files
assert os.path.exists(os.path.join(tmpdir, "jip-1.out"))
assert os.path.exists(os.path.join(tmpdir, "jip-1.err"))
# and we should have one job in Done state in our database
# we do the query with a fresh session though
job = jip.db.get(1)
assert job is not None
assert job.state == jip.db.STATE_FAILED
def test_job_hierarchy_execution_with_pipes_no_dispatching(tmpdir):
tmpdir = str(tmpdir)
target_file = os.path.join(tmpdir, 'result')
# create the pipeline
p = jip.Pipeline()
a = p.job(dir=tmpdir).bash('echo "hello world"')
b = p.job(dir=tmpdir).bash('wc -w', output=target_file)
a | b
p.context(locals())
# create the jobs
jobs = jip.create_jobs(p)
assert len(jobs[0].pipe_to) == 1
assert len(jobs) == 2
# iterate the executions and pass the session so all jobs are stored
execs = 0
for e in jip.create_executions(jobs):
jip.run_job(e.job)
execs += 1
assert execs == 1
# now the file should be there
assert os.path.exists(target_file)
for j in jobs:
assert j.state == jip.db.STATE_DONE
# check the content of the output files
assert open(target_file).read().strip() == "2"
def test_job_hierarchy_execution_with_dispatching_fan_out(tmpdir):
tmpdir = str(tmpdir)
target_file = os.path.join(tmpdir, 'result')
# create the pipeline
p = jip.Pipeline()
a = p.job(dir=tmpdir).bash('echo "hello world"', output=target_file + ".1")
b = p.job(dir=tmpdir).bash('wc -w', output=target_file + ".2")
c = p.job(dir=tmpdir).bash('wc -l', output=target_file + ".3")
a | (b + c)
p.context(locals())
# create the jobs
jobs = jip.create_jobs(p)
# iterate the executions and pass the session so all jobs are stored
execs = 0
for e in jip.create_executions(jobs):
jip.run_job(e.job)
execs += 1
assert execs == 1
# now the file should be there
for j in jobs:
assert j.state == jip.db.STATE_DONE
# check the content of the output files
assert open(target_file + '.1').read().strip() == "hello world"
assert open(target_file + '.3').read().strip() == "1"
assert open(target_file + '.2').read().strip() == "2"
def test_gem_name_option_delegation_with_output_dir():
p = jip.Pipeline()
p.run('grape_gem_rnapipeline',
fastq='reads_1.fastq.gz',
index='index.gem',
annotation='gencode.gtf',
output_dir="mydir")
jobs = jip.create_jobs(p, validate=False)
ldir = os.getcwd()
j = os.path.join
assert len(jobs) == 2
assert jobs[0].configuration['index'].get() == j(ldir, 'index.gem')
assert jobs[0].configuration['fastq'].get() == j(ldir, 'reads_1.fastq.gz')
assert jobs[0].configuration['annotation'].get() == j(ldir, 'gencode.gtf')
assert jobs[0].configuration['quality'].get() == '33'
assert jobs[0].configuration['output_dir'].get() == "mydir"
assert jobs[0].configuration['name'].get() == 'reads'
assert jobs[0].configuration['bam'].get() == j(ldir, 'mydir/reads.bam')
assert jobs[0].configuration['bai'].get() == j(ldir, 'mydir/reads.bam.bai')
assert jobs[0].configuration['map'].get() == j(ldir, 'mydir/reads.map.gz')
assert jobs[1].configuration['input'].get() == j(ldir, 'mydir/reads.bam')
assert jobs[1].configuration['name'].get() == 'reads'
assert jobs[1].configuration['annotation'].get() == j(ldir, 'gencode.gtf')
assert jobs[1].configuration['output_dir'].get() == "mydir"
assert jobs[1].configuration['gtf'].get() == j(ldir, 'mydir/reads.gtf')
assert len(jobs[0].children) == 1
assert len(jobs[1].dependencies) == 1
assert jobs[0].children[0] == jobs[1]
def test_bash_tool_job_rendering():
p = Pipeline()
p.run('bash', cmd="testme", output="test.out")
jobs = create_jobs(p)
assert len(jobs) == 1
assert jobs[0].command == "(testme)> " + os.path.join(os.getcwd(),
'test.out')
def test_tool_name_with_local_context():
p = jip.Pipeline()
a = p.run('foo', input='Makefile')
p.context(locals())
jobs = jip.create_jobs(p, validate=False)
assert len(jobs) == 1
assert jobs[0].name == 'Makefile'
def test_pipeline_overwrites_pipeline_from_spec():
@jip.tool()
class MyTool:
def setup(self):
self.profile.threads = 2
self.profile.queue = "Org"
def get_command(self):
return "echo"
@jip.pipeline()
class MyPipeline:
def pipeline(self):
p = jip.Pipeline()
p.job(threads=3, queue="Yeah").run("MyTool")
return p
p = jip.Pipeline()
p.run("MyPipeline")
p.expand()
profile = jip.Profile(threads=10, queue="Test")
profile.specs["MyTool"] = jip.Profile(threads=5)
profile.apply_to_pipeline(p)
jobs = jip.create_jobs(p)
assert jobs[0].threads == 5
assert jobs[0].queue == "Yeah"
def test_pipeline_tool_defaults_global_job():
@jip.tool()
class MyTool:
def setup(self):
self.profile.threads = 2
self.profile.queue = "Org"
def get_command(self):
return "echo"
@jip.pipeline()
class MyPipeline:
def pipeline(self):
p = jip.Pipeline()
p.job(threads=3, queue="Intern").run("MyTool")
return p
p = jip.Pipeline()
p.run("MyPipeline")
p.expand()
profile = jip.Profile(threads=5, queue="yeah")
profile.specs["MyTool"] = jip.Profile()
profile.apply_to_pipeline(p)
jobs = jip.create_jobs(p)
assert jobs[0].threads == 3
assert jobs[0].queue == "Intern"
def test_pipeline_tool_spec_regexp():
@jip.tool()
class MyTool:
def get_command(self):
return "echo"
@jip.pipeline()
class MyPipeline:
def pipeline(self):
p = jip.Pipeline()
p.job(threads=3, queue="Intern").run("MyTool")
return p
p = jip.Pipeline()
p.run("MyPipeline")
p.expand()
profile = jip.Profile(threads=5, queue="yeah", priority="high")
profile.specs["My*"] = jip.Profile(threads=10, queue="rock")
profile.apply_to_pipeline(p)
jobs = jip.create_jobs(p)
assert jobs[0].threads == 10
assert jobs[0].queue == "rock"
assert jobs[0].priority == "high"
def test_tool_name_in_pipeline_context_with_custom_profile_and_custom_name():
@jip.tool()
class MyTool():
def validate(self):
self.job.name = "testtool"
def get_command(self):
return "echo"
@jip.pipeline()
class MyPipeline():
def validate(self):
self.name("thepipeline")
def pipeline(self):
p = jip.Pipeline()
p.job('Tool1').run('MyTool')
return p
p = jip.Pipeline()
p.run('MyPipeline')
p.expand()
profile = jip.Profile(name="customname")
jobs = jip.create_jobs(p, profile=profile)
assert len(jobs) == 1
assert jobs[0].name == "Tool1"
assert jobs[0].pipeline == "customname"
def test_tool_name_in_pipeline_context():
@jip.tool()
class MyTool:
def validate(self):
self.job.name = "testtool"
def get_command(self):
return "echo"
@jip.pipeline()
class MyPipeline:
def validate(self):
self.name("thepipeline")
def pipeline(self):
p = jip.Pipeline()
p.run("MyTool")
return p
p = jip.Pipeline()
p.run("MyPipeline")
p.expand()
jobs = jip.create_jobs(p)
assert len(jobs) == 1
assert jobs[0].name == "testtool"
assert jobs[0].pipeline == "thepipeline"
def test_pipeline_overwrites_pipeline_from_spec():
@jip.tool()
class MyTool():
def setup(self):
self.profile.threads = 2
self.profile.queue = "Org"
def get_command(self):
return "echo"
@jip.pipeline()
class MyPipeline():
def pipeline(self):
p = jip.Pipeline()
p.job(threads=3, queue="Yeah").run('MyTool')
return p
p = jip.Pipeline()
p.run('MyPipeline')
p.expand()
profile = jip.Profile(threads=10, queue="Test")
profile.specs['MyTool'] = jip.Profile(threads=5)
profile.apply_to_pipeline(p)
jobs = jip.create_jobs(p)
assert jobs[0].threads == 5
assert jobs[0].queue == "Yeah"
def test_job_hierarchy_job_group(tmpdir):
tmpdir = str(tmpdir)
target_file = os.path.join(tmpdir, 'result')
@jip.tool()
def merge():
"""\
Merge
usage:
merge --input <input>... [--output <output>]
Options:
--input <input>... The input
[default: stdin]
--output <output> The input
[default: stdout]
"""
return "cat ${input|else('-')} ${output|arg('> ')}"
# create the pipeline
p = jip.Pipeline()
a_1 = p.job(dir=tmpdir).bash('echo "hello spain"',
output=target_file + ".1")
a_2 = p.job(dir=tmpdir).bash('echo "hello world"',
output=target_file + ".2")
a_3 = p.job(dir=tmpdir).bash('echo "hello universe"',
output=target_file + ".3")
b = p.job(dir=tmpdir).run('merge', output=target_file)
b.input = [a_1, a_2, a_3]
(a_1 - a_2 - a_3 - b)
p.context(locals())
# create the jobs
jobs = jip.create_jobs(p)
assert len(jobs) == 4
assert len(jobs[0].dependencies) == 0
assert len(jobs[0].children) == 2
assert len(jobs[1].dependencies) == 1
assert len(jobs[1].children) == 2
assert len(jobs[2].dependencies) == 1
assert len(jobs[2].children) == 1
assert len(jobs[3].dependencies) == 3
print jobs[3].command
# iterate the executions and pass the session so all jobs are stored
execs = 0
for e in jip.create_executions(jobs):
jip.run_job(e.job)
execs += 1
assert execs == 1
# now the file should be there
for j in jobs:
assert j.state == jip.db.STATE_DONE
# check the content of the output files
assert open(target_file + '.1').read().strip() == "hello spain"
assert open(target_file + '.2').read().strip() == "hello world"
assert open(target_file + '.3').read().strip() == "hello universe"
assert open(target_file).read().strip() == "hello spain\n"\
"hello world\nhello universe"
def test_pipeline_tool_spec_regexp():
@jip.tool()
class MyTool():
def get_command(self):
return "echo"
@jip.pipeline()
class MyPipeline():
def pipeline(self):
p = jip.Pipeline()
p.job(threads=3, queue="Intern").run('MyTool')
return p
p = jip.Pipeline()
p.run('MyPipeline')
p.expand()
profile = jip.Profile(threads=5, queue="yeah", priority="high")
profile.specs['My*'] = jip.Profile(threads=10, queue="rock")
profile.apply_to_pipeline(p)
jobs = jip.create_jobs(p)
assert jobs[0].threads == 10
assert jobs[0].queue == "rock"
assert jobs[0].priority == "high"
def test_gem_name_option_delegation_with_output_dir():
p = jip.Pipeline()
p.run('grape_gem_rnapipeline', fastq='reads_1.fastq.gz', index='index.gem',
annotation='gencode.gtf', output_dir="mydir")
jobs = jip.create_jobs(p, validate=False)
ldir = os.getcwd()
j = os.path.join
assert len(jobs) == 2
assert jobs[0].configuration['index'].get() == j(ldir, 'index.gem')
assert jobs[0].configuration['fastq'].get() == j(ldir, 'reads_1.fastq.gz')
assert jobs[0].configuration['annotation'].get() == j(ldir, 'gencode.gtf')
assert jobs[0].configuration['quality'].get() == '33'
assert jobs[0].configuration['output_dir'].get() == "mydir"
assert jobs[0].configuration['name'].get() == 'reads'
assert jobs[0].configuration['bam'].get() == j(ldir, 'mydir/reads.bam')
assert jobs[0].configuration['bai'].get() == j(ldir, 'mydir/reads.bam.bai')
assert jobs[0].configuration['map'].get() == j(ldir, 'mydir/reads.map.gz')
assert jobs[1].configuration['input'].get() == j(ldir, 'mydir/reads.bam')
assert jobs[1].configuration['name'].get() == 'reads'
assert jobs[1].configuration['annotation'].get() == j(ldir, 'gencode.gtf')
assert jobs[1].configuration['output_dir'].get() == "mydir"
assert jobs[1].configuration['gtf'].get() == j(ldir, 'mydir/reads.gtf')
assert len(jobs[0].children) == 1
assert len(jobs[1].dependencies) == 1
assert jobs[0].children[0] == jobs[1]
def test_gem_pipeline_with_output_dir():
p = jip.Pipeline()
p.run('grape_gem_rnapipeline', fastq='reads_1.fastq.gz', genome='index.fa',
annotation='gencode.gtf', output_dir="mydir", max_matches='10', max_mismatches='4')
jobs = jip.create_jobs(p, validate=False)
ldir = os.getcwd()
j = os.path.join
assert len(jobs) == 4
assert jobs[2].configuration['index'].get() == j(ldir, 'index.gem')
assert jobs[2].configuration['fastq'].get() == j(ldir, 'reads_1.fastq.gz')
assert jobs[2].configuration['transcript_index'].get() == j(ldir, 'gencode.gtf.gem')
assert jobs[2].configuration['quality'].get() == '33'
assert jobs[2].configuration['output_dir'].get() == "mydir"
assert jobs[2].configuration['name'].get() == 'reads'
assert jobs[2].configuration['bam'].get() == j(ldir, 'mydir/reads.bam')
assert jobs[2].configuration['bai'].get() == j(ldir, 'mydir/reads.bam.bai')
assert jobs[2].configuration['map'].get() == j(ldir, 'mydir/reads.map.gz')
assert jobs[3].configuration['input'].get() == j(ldir, 'mydir/reads.bam')
assert jobs[3].configuration['name'].get() == 'reads'
assert jobs[3].configuration['annotation'].get() == j(ldir, 'gencode.gtf')
assert jobs[3].configuration['output_dir'].get() == "mydir"
assert jobs[3].configuration['output'].get() == j(ldir, 'mydir/reads.gtf')
assert len(jobs[0].children) == 2
assert len(jobs[1].dependencies) == 1
assert len(jobs[2].dependencies) == 2
assert len(jobs[3].dependencies) == 1
assert jobs[0].children[0] == jobs[1]
def test_embedded_pipelines():
@jip.tool()
class produce():
"""Produce a set of files
Usage:
produce --prefix <prefix> --number <number>
"""
def init(self):
self.add_output('output', '${prefix}.*', nargs="*")
def get_command(self):
return """
for x in $(seq ${number}); do
echo Hello $x > ${prefix}.$x;
done
"""
@jip.tool()
def consume():
"""Count something
Usage:
consume <input>
"""
return """cat ${input}"""
p = jip.Pipeline()
# produce n files
producer = p.run('produce', prefix='test', number=5)
# run after success dynamically
producer.on_success('consume', input=producer)
jobs = jip.create_jobs(p)
assert len(jobs) == 1
assert len(jobs[0].on_success) == 1
def test_setting_working_directory_to_sub():
cwd = os.getcwd()
p = jip.Pipeline()
# produce n files
p.job(dir="sub").run('produce', prefix='test', number=5)
jobs = jip.create_jobs(p)
assert jobs[0].working_directory == cwd + "/sub"
def test_embedded_pipelines():
@jip.tool()
class produce():
"""Produce a set of files
Usage:
produce --prefix <prefix> --number <number>
"""
def init(self):
self.add_output('output', '${prefix}.*', nargs="*")
def get_command(self):
return """
for x in $(seq ${number}); do
echo Hello $x > ${prefix}.$x;
done
"""
@jip.tool()
def consume():
"""Count something
Usage:
consume <input>
"""
return """cat ${input}"""
p = jip.Pipeline()
# produce n files
producer = p.run('produce', prefix='test', number=5)
# run after success dynamically
producer.on_success('consume', input=producer)
jobs = jip.create_jobs(p)
assert len(jobs) == 1
assert len(jobs[0].on_success) == 1
def test_embedded_options_are_absolute():
jip.scanner.add_folder("test/data/makeabs")
p = jip.Pipeline()
p.run('makeabs', infile="Makefile", output="result")
jobs = jip.create_jobs(p)
assert len(jobs) == 1
cwd = os.getcwd()
assert jobs[0].command == "(cat %s/Makefile)> %s/result" % (cwd, cwd)
def test_embedded_options_are_absolute():
jip.scanner.add_folder("test/data/makeabs")
p = jip.Pipeline()
p.run('makeabs', infile="Makefile", output="result")
jobs = jip.create_jobs(p)
assert len(jobs) == 1
cwd = os.getcwd()
assert jobs[0].command == "(cat %s/Makefile)> %s/result" % (cwd, cwd)
def test_ouput_file_update_on_insert():
p = jip.Pipeline()
p.run('bash', cmd="touch ${outfile}", outfile="A.txt")
jobs = jip.create_jobs(p)
jip.db.save(jobs)
assert len(jobs[0].out_files) == 1
assert jobs[0].out_files[0].job == jobs[0]
assert jip.db.create_session().query(jip.db.OutputFile).count() == 1
def test_hello_world_py_cls(tmpdir):
tmpdir = str(tmpdir)
jip.scanner.add_module('examples/hello_world/hello_world.py')
jip.scanner.scan_modules()
p = jip.Pipeline()
p.job(dir=tmpdir).run('cls_hello_world_py')
jobs = jip.create_jobs(p)
assert len(jobs) == 1
def test_tool_script_and_injected_attributes(funcname):
script = ScriptTool.from_string(
"""#!/bin/bash
#Simple tool
#
#%%begin validate
assert '%s' in locals(), "Injected function %s not found"
#%%end
echo
""" % (funcname, funcname))
assert script is not None
p = jip.Pipeline()
p.run(script)
p.expand()
jip.create_jobs(p)
def test_setting_working_directory_cwd_with_profile():
cwd = os.getcwd()
p = jip.Pipeline()
# produce n files
p.run('produce', prefix='test', number=5)
profile = jip.profiles.Profile()
jobs = jip.create_jobs(p, profile=profile)
assert jobs[0].working_directory == cwd
def test_setting_working_directory_cwd_with_profile():
cwd = os.getcwd()
p = jip.Pipeline()
# produce n files
p.run('produce', prefix='test', number=5)
profile = jip.profiles.Profile()
jobs = jip.create_jobs(p, profile=profile)
assert jobs[0].working_directory == cwd
def test_hello_world_py_cls(tmpdir):
tmpdir = str(tmpdir)
jip.scanner.add_module('examples/hello_world/hello_world.py')
jip.scanner.scan_modules()
p = jip.Pipeline()
p.job(dir=tmpdir).run('cls_hello_world_py')
jobs = jip.create_jobs(p)
assert len(jobs) == 1
def test_setting_working_directory_to_sub_with_profile_and_render_output():
cwd = os.getcwd()
p = jip.Pipeline()
# produce n files
p.job(dir="sub").bash('hostname ${outfile}', outfile="a.txt")
profile = jip.profiles.Profile()
jobs = jip.create_jobs(p, profile=profile)
assert jobs[0].working_directory == cwd + "/sub"
assert jobs[0].configuration['outfile'].get() == cwd + "/sub/a.txt"
def test_setting_working_directory_to_sub_with_profile_and_render_output():
cwd = os.getcwd()
p = jip.Pipeline()
# produce n files
p.job(dir="sub").bash('hostname ${outfile}', outfile="a.txt")
profile = jip.profiles.Profile()
jobs = jip.create_jobs(p, profile=profile)
assert jobs[0].working_directory == cwd + "/sub"
assert jobs[0].configuration['outfile'].get() == cwd + "/sub/a.txt"
def test_job_hierarchy_execution_with_pipes_and_dispatching(tmpdir):
print ">>>", tmpdir
tmpdir = str(tmpdir)
target_file = os.path.join(tmpdir, 'result')
db_file = os.path.join(tmpdir, "test.db")
# create a JIP database and a session
jip.db.init(db_file)
session = jip.db.create_session()
# create the cluster instance
c = cl.LocalCluster()
# create the pipeline
p = jip.Pipeline()
a = p.job(dir=tmpdir).bash('echo "hello world"', output="${target_file}.1")
b = p.job(dir=tmpdir).bash('wc -w ${input}',
input=a,
output="${target_file}.2")
l = p.job(dir=tmpdir).bash('echo "Other" > ${target_file}.3')
a | b
p.context(locals())
# create the jobs
jobs = jip.create_jobs(p)
assert len(jobs) == 3
# iterate the executions and pass the session so all jobs are stored
execs = 0
for e in jip.create_executions(jobs, save=True):
jip.submit_job(e.job, save=True, cluster=c)
execs += 1
assert execs == 2
c.wait()
# now the file should be there
assert os.path.exists(target_file + ".1")
assert os.path.exists(target_file + ".2")
assert os.path.exists(target_file + ".3")
# we should also have the log files
assert os.path.exists(os.path.join(tmpdir, "jip-1.out"))
assert os.path.exists(os.path.join(tmpdir, "jip-1.err"))
assert os.path.exists(os.path.join(tmpdir, "jip-3.out"))
assert os.path.exists(os.path.join(tmpdir, "jip-3.err"))
# and we should have one job in Done state in our database
# we do the query with a fresh session though
find = jip.db.get
assert find(1).state == jip.db.STATE_DONE
assert find(2).state == jip.db.STATE_DONE
assert find(3).state == jip.db.STATE_DONE
# check the content of the output files
assert open(target_file + ".1").read() == "hello world\n"
assert open(target_file + ".2").read().strip() == "2"
assert open(target_file + ".3").read() == "Other\n"
def test_job_hierarchy_execution_with_pipes_and_dispatching(tmpdir):
print ">>>", tmpdir
tmpdir = str(tmpdir)
target_file = os.path.join(tmpdir, 'result')
db_file = os.path.join(tmpdir, "test.db")
# create a JIP database and a session
jip.db.init(db_file)
session = jip.db.create_session()
# create the cluster instance
c = cl.LocalCluster()
# create the pipeline
p = jip.Pipeline()
a = p.job(dir=tmpdir).bash('echo "hello world"',
output="${target_file}.1")
b = p.job(dir=tmpdir).bash('wc -w ${input}',
input=a, output="${target_file}.2")
l = p.job(dir=tmpdir).bash('echo "Other" > ${target_file}.3')
a | b
p.context(locals())
# create the jobs
jobs = jip.create_jobs(p)
assert len(jobs) == 3
# iterate the executions and pass the session so all jobs are stored
execs = 0
for e in jip.create_executions(jobs, save=True):
jip.submit_job(e.job, save=True, cluster=c)
execs += 1
assert execs == 2
c.wait()
# now the file should be there
assert os.path.exists(target_file + ".1")
assert os.path.exists(target_file + ".2")
assert os.path.exists(target_file + ".3")
# we should also have the log files
assert os.path.exists(os.path.join(tmpdir, "jip-1.out"))
assert os.path.exists(os.path.join(tmpdir, "jip-1.err"))
assert os.path.exists(os.path.join(tmpdir, "jip-3.out"))
assert os.path.exists(os.path.join(tmpdir, "jip-3.err"))
# and we should have one job in Done state in our database
# we do the query with a fresh session though
find = jip.db.get
assert find(1).state == jip.db.STATE_DONE
assert find(2).state == jip.db.STATE_DONE
assert find(3).state == jip.db.STATE_DONE
# check the content of the output files
assert open(target_file + ".1").read() == "hello world\n"
assert open(target_file + ".2").read().strip() == "2"
assert open(target_file + ".3").read() == "Other\n"
def test_input_file_update_on_insert(tmpdir):
db_file = os.path.join(str(tmpdir), "test.db")
jip.db.init(db_file)
p = jip.Pipeline()
p.run('bash', cmd="touch ${input}", input="A.txt")
jobs = jip.create_jobs(p, validate=False)
jip.db.save(jobs)
assert len(jobs[0].in_files) == 1
assert jobs[0].in_files[0].job == jobs[0]
assert jip.db.create_session().query(jip.db.InputFile).count() == 1
def test_tool_class_and_injected_attributes(funcname):
called = []
@jip.tool()
class MyTool():
def validate(self):
# call injected functions
assert hasattr(self, funcname), "Injected function %s "\
"not found" % funcname
called.append(True)
def get_command(self):
return "echo"
p = jip.Pipeline()
p.run('MyTool')
p.expand()
jip.create_jobs(p)
assert len(called) >= 1
def test_input_file_delete_on_job_delete(tmpdir):
db_file = os.path.join(str(tmpdir), "test.db")
jip.db.init(db_file)
p = jip.Pipeline()
p.run('bash', cmd="touch ${input}", input="A.txt")
jobs = jip.create_jobs(p, validate=False)
jip.db.save(jobs)
assert jip.db.create_session().query(jip.db.InputFile).count() == 1
jip.db.delete(jobs)
assert jip.db.create_session().query(jip.db.Job).count() == 0
def test_storing_embdded_pipeline():
p = jip.Pipeline()
p.run('embedded')
# create the jobs
jobs = jip.create_jobs(p)
assert len(jobs) == 1
assert jobs[0].on_success is not None
emb = jobs[0].on_success
assert len(emb) == 1
data = pickle.dumps(emb)
assert data is not None
def test_multiplex_with_stream():
p = jip.Pipeline()
first = p.bash("cat ${input}", input=['A', 'B'])
second = p.bash("wc -l")
first | second
p.expand(validate=False)
assert len(p) == 4
jobs = jip.create_jobs(p, validate=False)
assert len(jobs) == 4
execs = jip.create_executions(jobs)
assert len(execs) == 2
def test_input_file_update_on_insert(db, tmpdir):
if not db.startswith('mysql'):
db = os.path.join(str(tmpdir), db)
jip.db.init(db)
p = jip.Pipeline()
p.run('bash', cmd="touch ${input}", input="A.txt")
jobs = jip.create_jobs(p, validate=False)
jip.db.save(jobs)
assert len(jobs[0].in_files) == 1
assert jobs[0].in_files[0].job == jobs[0]
assert jip.db.create_session().query(jip.db.InputFile).count() == 1
def test_ouput_file_update_on_insert(db, tmpdir):
if not db.startswith('mysql'):
db = os.path.join(str(tmpdir), db)
jip.db.init(db)
p = jip.Pipeline()
p.run('bash', cmd="touch ${outfile}", outfile="A.txt")
jobs = jip.create_jobs(p)
jip.db.save(jobs)
assert len(jobs[0].out_files) == 1
assert jobs[0].out_files[0].job == jobs[0]
assert jip.db.create_session().query(jip.db.OutputFile).count() == 1
def test_multiplex_with_stream():
p = jip.Pipeline()
first = p.bash("cat ${input}", input=['A', 'B'])
second = p.bash("wc -l")
first | second
p.expand(validate=False)
assert len(p) == 4
jobs = jip.create_jobs(p, validate=False)
assert len(jobs) == 4
execs = jip.create_executions(jobs)
assert len(execs) == 2
def test_embedded_pipelines_stage_one(tmpdir):
tmpdir = str(tmpdir)
# laod teh embedded example
jip.scanner.add_module('examples/embedded_submission/embedded.py')
jip.scanner.scan_modules()
p = jip.Pipeline()
p.job(dir=tmpdir).run('example_embedded')
jobs = jip.create_jobs(p)
assert jobs[0].configuration['prefix'] == 'test'
assert jobs[0].configuration['output'] == [os.path.join(tmpdir, 'test.*')]
assert len(jobs) == 1
def test_input_file_query(db, tmpdir):
if not db.startswith('mysql'):
db = os.path.join(str(tmpdir), db)
jip.db.init(db)
p = jip.Pipeline()
p.run('bash', cmd="touch ${input}", input="A.txt")
abspath = os.path.join(os.getcwd(), "A.txt")
jobs = jip.create_jobs(p, validate=False)
jip.db.save(jobs)
job = jip.db.query_by_files(inputs=abspath)
assert job.count() == 1
assert job.one() == jobs[0]
def test_module_decorator():
if os.environ.get('GRAPE_HOME'):
del os.environ['GRAPE_HOME']
@module([("gemtools", "1.6.1")])
@jip.tool('gemtools_test')
class gemtools(object):
"""
The GEM Indexer tool
Usage:
gem_index -i <genome> [-o <genome_index>] [-t <threads>] [--no-hash]
Options:
--help Show this help message
-o, --output <genome_index> The output GEM index file [default: ${input|ext}.gem]
-t, --threads <threads> The number of execution threads [default: 1]
--no-hash Do not produce the hash file [default: false]
Inputs:
-i, --input <genome> The fasta file for the genome
"""
def validate(self):
return True
def get_command(self):
return "bash", "gemtools index ${options()}"
j = os.path.join
p = jip.Pipeline()
p.run('gemtools_test', input='genome.fa')
with pytest.raises(jip.ValidationError) as excinfo:
jip.create_jobs(p)
assert excinfo.value.message == 'GRAPE_HOME not defined. Please set the GRAPE_HOME environment variable!'