def test_pipeline_with_commands(self):
"""
Pipeline: define and run pipeline with commands
"""
# Define a task
# Echoes/appends text to a file
class Echo(PipelineTask):
def init(self, f, s):
pass
def setup(self):
self.add_cmd(
PipelineCommandWrapper("Echo text to file", "echo",
self.args.s, ">>", self.args.f))
def output(self):
return self.args.f
# Build the pipeline
ppl = Pipeline()
task1 = Echo("Write item1", "out.txt", "item1")
task2 = Echo("Write item2", task1.output(), "item2")
ppl.add_task(task2, requires=(task1, ))
# Run the pipeline
exit_status = ppl.run(working_dir=self.working_dir)
# Check the outputs
self.assertEqual(exit_status, 0)
out_file = os.path.join(self.working_dir, "out.txt")
self.assertTrue(os.path.exists(out_file))
self.assertEqual(open(out_file, 'r').read(), "item1\nitem2\n")
def test_simple_pipeline(self):
"""
Pipeline: define and run a simple pipeline
"""
# Define a reusable task
# Appends item to a list
class Append(PipelineTask):
def init(self, l, s):
self.l = list()
def setup(self):
for item in self.args.l:
self.l.append(item)
self.l.append(self.args.s)
def output(self):
return self.l
# Build the pipeline
ppl = Pipeline()
task1 = Append("Append 1", (), "item1")
task2 = Append("Append 2", task1.output(), "item2")
ppl.add_task(task2, requires=(task1, ))
# Run the pipeline
exit_status = ppl.run(working_dir=self.working_dir)
# Check the outputs
self.assertEqual(exit_status, 0)
self.assertEqual(task1.output(), ["item1"])
self.assertEqual(task2.output(), ["item1", "item2"])
def test_pipeline_stops_on_task_failure(self):
"""
Pipeline: check pipeline stops on task failure
"""
# Define a reusable task
# Appends item to a list
class Append(PipelineTask):
def init(self, l, s):
self.l = list()
def setup(self):
for item in self.args.l:
self.l.append(item)
self.l.append(self.args.s)
def output(self):
return self.l
# Define a task that always fails
class Failure(PipelineTask):
def init(self):
pass
def setup(self):
self.fail(message="Automatic fail")
def output(self):
return None
# Build the pipeline
ppl = Pipeline()
task1 = Append("Append 1", (), "item1")
task2 = Failure("Failing task")
task3 = Append("Append 3", task1.output(), "item3")
ppl.add_task(task2, requires=(task1, ))
ppl.add_task(task3, requires=(task2, ))
# Run the pipeline
exit_status = ppl.run(working_dir=self.working_dir)
# Check the outputs
self.assertEqual(exit_status, 1)
self.assertEqual(task1.output(), ["item1"])
self.assertEqual(task2.exit_code, 1)
self.assertEqual(task3.output(), [])
def test_pipeline_working_dir_is_respected(self):
"""
Pipeline: check pipeline respects the working directory
"""
# Define tasks
# Echoes/appends text to a file via shell command
class Echo(PipelineTask):
def init(self, f, s):
pass
def setup(self):
self.add_cmd(
PipelineCommandWrapper("Echo text to file", "echo",
self.args.s, ">>", self.args.f))
def output(self):
return self.args.f
# Writes text to a file via Python
class Print(PipelineTask):
def init(self, f, s):
pass
def setup(self):
with open(self.args.f, 'a') as fp:
fp.write("%s\n" % self.args.s)
def output(self):
return self.args.f
# Build the pipeline
ppl = Pipeline()
task1 = Echo("Echo item1", "out.txt", "item1")
task2 = Print("Print item2", task1.output(), "item2")
ppl.add_task(task2, requires=(task1, ))
# Run the pipeline
exit_status = ppl.run(working_dir=self.working_dir)
# Check the outputs
self.assertEqual(exit_status, 0)
out_file = os.path.join(self.working_dir, "out.txt")
self.assertTrue(os.path.exists(out_file))
self.assertEqual(open(out_file, 'r').read(), "item1\nitem2\n")
def filter_empty_fastqs(self, *fastqs):
filtered_fastqs = list()
for fq in fastqs:
if nreads(fq) > 0:
print("%s" % fq)
filtered_fastqs.append(fq)
return filtered_fastqs
def finish(self):
for result in self.result():
for fq in result:
self.output.fastqs.append(fq)
if __name__ == "__main__":
# Command line
p = argparse.ArgumentParser()
p.add_argument("fastqs", nargs='+', metavar="FASTQ")
args = p.parse_args()
# Make and run a pipeline
ppl = Pipeline()
filter_empty_fastqs = FilterEmptyFastqs("Filter empty Fastqs", args.fastqs)
run_fastqc = RunFastqc("Run Fastqc", filter_empty_fastqs.output.fastqs,
os.getcwd())
ppl.add_task(filter_empty_fastqs)
ppl.add_task(run_fastqc, requires=(filter_empty_fastqs, ))
ppl.run()
print(run_fastqc.output())