def test_transform_with_missing_formatter_args_b(self): test_pipeline = Pipeline("test") test_pipeline.originate(task_func=generate_initial_files, output=[os.path.join(tempdir, ff + ".tmp") for ff in "abcd"])\ .mkdir(tempdir) test_pipeline.transform( task_func=transform_with_missing_formatter_args, input=generate_initial_files, filter=formatter(), output="{path[0]}/{basename[0]}.task1", extras=['echo {dynamic_message} > {some_file}']) s = StringIO() test_pipeline.printout(s, [transform_with_missing_formatter_args], verbose=4, wrap_width=10000, pipeline="test") self.assertIn("Unmatched field {dynamic_message}", s.getvalue()) # log to stream s = StringIO() logger = t_stream_logger(s) test_pipeline.run([transform_with_missing_formatter_args], verbose=5, pipeline="test", logger=logger) self.assertIn("Unmatched field {dynamic_message}", s.getvalue())
def test_newstyle_collate(self): """ As above but create pipeline on the fly using object orientated syntax rather than decorators """ # # Create pipeline on the fly, joining up tasks # test_pipeline = Pipeline("test") test_pipeline.originate(task_func = generate_initial_files, output = original_files)\ .mkdir(tempdir, tempdir+"/test") test_pipeline.subdivide( task_func = split_fasta_file, input = generate_initial_files, filter = regex(r".*\/original_(\d+).fa"), # match original files output = [tempdir + r"/files.split.\1.success", # flag file for each original file tempdir + r"/files.split.\1.*.fa"], # glob pattern extras = [r"\1"])\ .posttask(lambda: sys.stderr.write("\tSplit into %d files each\n" % JOBS_PER_TASK)) test_pipeline.transform(task_func = align_sequences, input = split_fasta_file, filter = suffix(".fa"), output = ".aln") \ .posttask(lambda: sys.stderr.write("\tSequences aligned\n")) test_pipeline.transform(task_func = percentage_identity, input = align_sequences, # find all results from align_sequences filter = suffix(".aln"), # replace suffix with: output = [r".pcid", # .pcid suffix for the result r".pcid_success"] # .pcid_success to indicate job completed )\ .posttask(lambda: sys.stderr.write("\t%Identity calculated\n")) test_pipeline.collate(task_func = combine_results, input = percentage_identity, filter = regex(r".*files.split\.(\d+)\.\d+.pcid"), output = [tempdir + r"/\1.all.combine_results", tempdir + r"/\1.all.combine_results_success"])\ .posttask(lambda: sys.stderr.write("\tResults recombined\n")) # # Cleanup, printout and run # self.cleanup_tmpdir() s = StringIO() test_pipeline.printout(s, [combine_results], verbose=5, wrap_width=10000) self.assertTrue( re.search('Job needs update:.*Missing files.*', s.getvalue(), re.DOTALL) is not None) test_pipeline.run(verbose=0)
def test_newstyle_collate(self): """ As above but create pipeline on the fly using object orientated syntax rather than decorators """ # # Create pipeline on the fly, joining up tasks # test_pipeline = Pipeline("test") test_pipeline.originate(task_func=generate_initial_files, output=original_files)\ .mkdir(tempdir, tempdir+"/test") test_pipeline.subdivide(task_func=split_fasta_file, input=generate_initial_files, # match original files filter=regex(r".*\/original_(\d+).fa"), output=[tempdir + r"/files.split.\1.success", # flag file for each original file tempdir + r"/files.split.\1.*.fa"], # glob pattern extras=[r"\1"])\ .posttask(lambda: sys.stderr.write("\tSplit into %d files each\n" % JOBS_PER_TASK)) test_pipeline.transform(task_func=align_sequences, input=split_fasta_file, filter=suffix(".fa"), output=".aln") \ .posttask(lambda: sys.stderr.write("\tSequences aligned\n")) test_pipeline.transform(task_func=percentage_identity, input=align_sequences, # find all results from align_sequences # replace suffix with: filter=suffix(".aln"), output=[r".pcid", # .pcid suffix for the result r".pcid_success"] # .pcid_success to indicate job completed )\ .posttask(lambda: sys.stderr.write("\t%Identity calculated\n")) test_pipeline.collate(task_func=combine_results, input=percentage_identity, filter=regex(r".*files.split\.(\d+)\.\d+.pcid"), output=[tempdir + r"/\1.all.combine_results", tempdir + r"/\1.all.combine_results_success"])\ .posttask(lambda: sys.stderr.write("\tResults recombined\n")) # # Cleanup, printout and run # self.cleanup_tmpdir() s = StringIO() test_pipeline.printout(s, [combine_results], verbose=5, wrap_width=10000) self.assertTrue(re.search( 'Job needs update:.*Missing files.*', s.getvalue(), re.DOTALL) is not None) test_pipeline.run(verbose=0)
def test_transform_with_missing_formatter_args_b(self): test_pipeline = Pipeline("test") test_pipeline.originate(task_func = generate_initial_files, output = [os.path.join(tempdir, ff + ".tmp") for ff in "abcd"])\ .mkdir(tempdir) test_pipeline.transform(task_func = transform_with_missing_formatter_args, input = generate_initial_files, filter = formatter(), output = "{path[0]}/{basename[0]}.task1", extras =['echo {dynamic_message} > {some_file}']) s = StringIO() test_pipeline.printout(s, [transform_with_missing_formatter_args], verbose=4, wrap_width = 10000, pipeline= "test") self.assertIn("Missing key = {dynamic_message}", s.getvalue()) #log to stream s = StringIO() logger = t_stream_logger(s) test_pipeline.run([transform_with_missing_formatter_args], verbose=5, pipeline= "test", logger=logger) self.assertIn("Missing key = {dynamic_message}", s.getvalue())