def test_pipeline_with_commands(self): """ Pipeline: define and run pipeline with commands """ # Define a task # Echoes/appends text to a file class Echo(PipelineTask): def init(self, f, s): pass def setup(self): self.add_cmd( PipelineCommandWrapper("Echo text to file", "echo", self.args.s, ">>", self.args.f)) def output(self): return self.args.f # Build the pipeline ppl = Pipeline() task1 = Echo("Write item1", "out.txt", "item1") task2 = Echo("Write item2", task1.output(), "item2") ppl.add_task(task2, requires=(task1, )) # Run the pipeline exit_status = ppl.run(working_dir=self.working_dir) # Check the outputs self.assertEqual(exit_status, 0) out_file = os.path.join(self.working_dir, "out.txt") self.assertTrue(os.path.exists(out_file)) self.assertEqual(open(out_file, 'r').read(), "item1\nitem2\n")
def test_simple_pipeline(self): """ Pipeline: define and run a simple pipeline """ # Define a reusable task # Appends item to a list class Append(PipelineTask): def init(self, l, s): self.l = list() def setup(self): for item in self.args.l: self.l.append(item) self.l.append(self.args.s) def output(self): return self.l # Build the pipeline ppl = Pipeline() task1 = Append("Append 1", (), "item1") task2 = Append("Append 2", task1.output(), "item2") ppl.add_task(task2, requires=(task1, )) # Run the pipeline exit_status = ppl.run(working_dir=self.working_dir) # Check the outputs self.assertEqual(exit_status, 0) self.assertEqual(task1.output(), ["item1"]) self.assertEqual(task2.output(), ["item1", "item2"])
def test_pipeline_stops_on_task_failure(self): """ Pipeline: check pipeline stops on task failure """ # Define a reusable task # Appends item to a list class Append(PipelineTask): def init(self, l, s): self.l = list() def setup(self): for item in self.args.l: self.l.append(item) self.l.append(self.args.s) def output(self): return self.l # Define a task that always fails class Failure(PipelineTask): def init(self): pass def setup(self): self.fail(message="Automatic fail") def output(self): return None # Build the pipeline ppl = Pipeline() task1 = Append("Append 1", (), "item1") task2 = Failure("Failing task") task3 = Append("Append 3", task1.output(), "item3") ppl.add_task(task2, requires=(task1, )) ppl.add_task(task3, requires=(task2, )) # Run the pipeline exit_status = ppl.run(working_dir=self.working_dir) # Check the outputs self.assertEqual(exit_status, 1) self.assertEqual(task1.output(), ["item1"]) self.assertEqual(task2.exit_code, 1) self.assertEqual(task3.output(), [])
def test_pipeline_working_dir_is_respected(self): """ Pipeline: check pipeline respects the working directory """ # Define tasks # Echoes/appends text to a file via shell command class Echo(PipelineTask): def init(self, f, s): pass def setup(self): self.add_cmd( PipelineCommandWrapper("Echo text to file", "echo", self.args.s, ">>", self.args.f)) def output(self): return self.args.f # Writes text to a file via Python class Print(PipelineTask): def init(self, f, s): pass def setup(self): with open(self.args.f, 'a') as fp: fp.write("%s\n" % self.args.s) def output(self): return self.args.f # Build the pipeline ppl = Pipeline() task1 = Echo("Echo item1", "out.txt", "item1") task2 = Print("Print item2", task1.output(), "item2") ppl.add_task(task2, requires=(task1, )) # Run the pipeline exit_status = ppl.run(working_dir=self.working_dir) # Check the outputs self.assertEqual(exit_status, 0) out_file = os.path.join(self.working_dir, "out.txt") self.assertTrue(os.path.exists(out_file)) self.assertEqual(open(out_file, 'r').read(), "item1\nitem2\n")
def filter_empty_fastqs(self, *fastqs): filtered_fastqs = list() for fq in fastqs: if nreads(fq) > 0: print("%s" % fq) filtered_fastqs.append(fq) return filtered_fastqs def finish(self): for result in self.result(): for fq in result: self.output.fastqs.append(fq) if __name__ == "__main__": # Command line p = argparse.ArgumentParser() p.add_argument("fastqs", nargs='+', metavar="FASTQ") args = p.parse_args() # Make and run a pipeline ppl = Pipeline() filter_empty_fastqs = FilterEmptyFastqs("Filter empty Fastqs", args.fastqs) run_fastqc = RunFastqc("Run Fastqc", filter_empty_fastqs.output.fastqs, os.getcwd()) ppl.add_task(filter_empty_fastqs) ppl.add_task(run_fastqc, requires=(filter_empty_fastqs, )) ppl.run() print(run_fastqc.output())