def test_newstyle_ruffus (self):
test_pipeline = Pipeline("test")
test_pipeline.files(create_random_numbers, None, tempdir + "random_numbers.list")\
.follows(mkdir(tempdir))
test_pipeline.split(task_func = step_4_split_numbers_into_chunks,
input = tempdir + "random_numbers.list",
output = tempdir + "*.chunks")\
.follows(create_random_numbers)
test_pipeline.transform(task_func = step_5_calculate_sum_of_squares,
input = step_4_split_numbers_into_chunks,
filter = suffix(".chunks"),
output = ".sums")
test_pipeline.merge(task_func = step_6_calculate_variance, input = step_5_calculate_sum_of_squares, output = os.path.join(tempdir, "variance.result"))\
.posttask(lambda: sys.stdout.write(" hooray\n"))\
.posttask(print_hooray_again, print_whoppee_again, touch_file(os.path.join(tempdir, "done")))
test_pipeline.run(multiprocess = 50, verbose = 0)
output_file = os.path.join(tempdir, "variance.result")
if not os.path.exists (output_file):
raise Exception("Missing %s" % output_file)
def test_newstyle_ruffus(self):
test_pipeline = Pipeline("test")
test_pipeline.files(create_random_numbers, None, tempdir + "random_numbers.list")\
.follows(mkdir(tempdir))
test_pipeline.split(task_func = step_4_split_numbers_into_chunks,
input = tempdir + "random_numbers.list",
output = tempdir + "*.chunks")\
.follows(create_random_numbers)
test_pipeline.transform(task_func=step_5_calculate_sum_of_squares,
input=step_4_split_numbers_into_chunks,
filter=suffix(".chunks"),
output=".sums")
test_pipeline.merge(task_func = step_6_calculate_variance, input = step_5_calculate_sum_of_squares, output = os.path.join(tempdir, "variance.result"))\
.posttask(lambda: sys.stdout.write(" hooray\n"))\
.posttask(print_hooray_again, print_whoppee_again, touch_file(os.path.join(tempdir, "done")))
test_pipeline.run(multiprocess=50, verbose=0)
output_file = os.path.join(tempdir, "variance.result")
if not os.path.exists(output_file):
raise Exception("Missing %s" % output_file)
def test_newstyle_task(self):
test_pipeline = Pipeline("test")
test_pipeline.files(task1, [[None, tempdir + "a.1"], [None, tempdir + "b.1"]])\
.follows(mkdir(tempdir))
test_pipeline.files(task2, [[None, tempdir + "c.1"], [None, tempdir + "d.1"]])\
.follows(mkdir(tempdir))
test_pipeline.transform(task_func=task3,
input=task1,
filter=regex(r"(.+)"),
replace_inputs=ruffus.inputs(
((r"\1"), task2,
"test_transform_inputs.*y")),
output=r"\1.output")
test_pipeline.merge(task4, (task3), tempdir + "final.output")
test_pipeline.run([task4], multiprocess=10, verbose=0)
correct_output = "{tempdir}a.1.output:test_transform_inputs.py,{tempdir}a.1,{tempdir}c.1,{tempdir}d.1;{tempdir}b.1.output:test_transform_inputs.py,{tempdir}b.1,{tempdir}c.1,{tempdir}d.1;".format(
tempdir=tempdir)
with open(tempdir + "final.output") as ff:
real_output = ff.read()
self.assertEqual(correct_output, real_output)
def test_newstyle_task(self):
test_pipeline = Pipeline("test")
test_pipeline.files(task1, a)
save_to_str_logger = t_save_to_str_logger()
test_pipeline.run(multiprocess=10, logger=save_to_str_logger, verbose=1)
self.assertTrue("@files() was empty" in save_to_str_logger.warning_str)
print("\n Warning printed out correctly", file=sys.stderr)
def test_newstyle_task (self):
test_pipeline = Pipeline("test")
test_pipeline.files(task1, a)
save_to_str_logger = t_save_to_str_logger()
test_pipeline.run(multiprocess = 10,
logger = save_to_str_logger,
verbose = 1)
self.assertTrue("@files() was empty" in save_to_str_logger.warning_str)
print("\n Warning printed out correctly", file=sys.stderr)
def test_newstyle_ruffus (self):
test_pipeline = Pipeline("test")
test_pipeline.follows(setup_simulation_data, mkdir(gene_data_dir, simulation_data_dir))
test_pipeline.files(gwas_simulation, generate_simulation_params)\
.follows(setup_simulation_data)\
.follows(mkdir(working_dir, os.path.join(working_dir, "simulation_results")))
test_pipeline.collate(statistical_summary, gwas_simulation, regex(r"simulation_results/(\d+).\d+.simulation_res"), r"\1.mean")\
.posttask(lambda : sys.stdout.write("\nOK\n"))
test_pipeline.run(multiprocess = 50, verbose = 0)
for oo in "000.mean", "001.mean":
results_file_name = os.path.join(working_dir, oo)
if not os.path.exists(results_file_name):
raise Exception("Missing %s" % results_file_name)
def test_newstyle_ruffus(self):
test_pipeline = Pipeline("test")
test_pipeline.follows(setup_simulation_data,
mkdir(gene_data_dir, simulation_data_dir))
test_pipeline.files(gwas_simulation, generate_simulation_params)\
.follows(setup_simulation_data)\
.follows(mkdir(working_dir, os.path.join(working_dir, "simulation_results")))
test_pipeline.collate(statistical_summary, gwas_simulation, regex(r"simulation_results/(\d+).\d+.simulation_res"), r"\1.mean")\
.posttask(lambda: sys.stdout.write("\nOK\n"))
test_pipeline.run(multiprocess=50, verbose=0)
for oo in "000.mean", "001.mean":
results_file_name = os.path.join(working_dir, oo)
if not os.path.exists(results_file_name):
raise Exception("Missing %s" % results_file_name)
def test_newstyle_task (self):
"""
Same as above but construct a new pipeline on the fly without decorators
"""
test_pipeline = Pipeline("test")
test_pipeline.files(task1, None, tempdir + 'a.1')\
.follows(mkdir(tempdir))
test_pipeline.transform(task_func = task2,
input = task1,
filter = regex(r".*"),
output = tempdir + 'b.1')
test_pipeline.files(task3, task2, tempdir + 'c.1')
test_pipeline.files(task4, [[None, tempdir + 'd.1'], [None, tempdir + 'e.1']])\
.follows(task3)
test_pipeline.files(task5, task4, tempdir + "f.1")
test_pipeline.run(multiprocess = 10, verbose = 0)
def test_newstyle_task(self):
"""
Same as above but construct a new pipeline on the fly without decorators
"""
test_pipeline = Pipeline("test")
test_pipeline.files(task1, None, tempdir + 'a.1')\
.follows(mkdir(tempdir))
test_pipeline.transform(task_func=task2,
input=task1,
filter=regex(r".*"),
output=tempdir + 'b.1')
test_pipeline.files(task3, task2, tempdir + 'c.1')
test_pipeline.files(task4, [[None, tempdir + 'd.1'], [None, tempdir + 'e.1']])\
.follows(task3)
test_pipeline.files(task5, task4, tempdir + "f.1")
test_pipeline.run(multiprocess=10, verbose=0)
def test_newstyle_task(self):
test_pipeline = Pipeline("test")
test_pipeline.files(task1, [[None, tempdir + "a.1"], [None, tempdir + "b.1"]])\
.follows(mkdir(tempdir))
test_pipeline.files(task2, [[None, tempdir + "c.1"], [None, tempdir + "d.1"]])\
.follows(mkdir(tempdir))
test_pipeline.transform(task_func=task3,
input=task1,
filter=regex(r"(.+)"),
replace_inputs=ruffus.inputs(
((r"\1"), task2, "test_transform_inputs.*y")),
output=r"\1.output")
test_pipeline.merge(task4, (task3), tempdir + "final.output")
test_pipeline.run([task4], multiprocess=10, verbose=0)
correct_output = "{tempdir}a.1.output:test_transform_inputs.py,{tempdir}a.1,{tempdir}c.1,{tempdir}d.1;{tempdir}b.1.output:test_transform_inputs.py,{tempdir}b.1,{tempdir}c.1,{tempdir}d.1;".format(
tempdir=tempdir)
with open(tempdir + "final.output") as ff:
real_output = ff.read()
self.assertEqual(correct_output, real_output)
input = task1,
filter = suffix(".1"),
output = ".2") \
.posttask(lambda: do_write(test_file, "Task 2 Done\n"))
test_pipeline.transform(task3, task2, regex('(.*).2'), inputs([r"\1.2", tempdir + "a.1"]), r'\1.3')\
.posttask(lambda: do_write(test_file, "Task 3 Done\n"))
test_pipeline.transform(task4, tempdir + "*.1", suffix(".1"), ".4")\
.follows(task1)\
.posttask(lambda: do_write(test_file, "Task 4 Done\n"))\
.jobs_limit(1)
test_pipeline.files(task5, None, tempdir + 'a.5')\
.follows(mkdir(tempdir))\
.posttask(lambda: do_write(test_file, "Task 5 Done\n"))
test_pipeline.merge(task_func = task6,
input = [task3, task4, task5],
output = tempdir + "final.6")\
.follows(task3, task4, task5, ) \
.posttask(lambda: do_write(test_file, "Task 6 Done\n"))
def check_job_order_correct(filename):
"""
1 -> 2 -> 3 ->
-> 4 ->
5 -> 6
"""
input = task1,
filter = suffix(".1"),
output = ".2") \
.posttask(lambda: do_write(test_file, "Task 2 Done\n"))
test_pipeline.transform(task3, task2, regex('(.*).2'), inputs([r"\1.2", tempdir + "a.1"]), r'\1.3')\
.posttask(lambda: do_write(test_file, "Task 3 Done\n"))
test_pipeline.transform(task4, tempdir + "*.1", suffix(".1"), ".4")\
.follows(task1)\
.posttask(lambda: do_write(test_file, "Task 4 Done\n"))\
.jobs_limit(1)
test_pipeline.files(task5, None, tempdir + 'a.5')\
.follows(mkdir(tempdir))\
.posttask(lambda: do_write(test_file, "Task 5 Done\n"))
test_pipeline.merge(task_func = task6,
input = [task3, task4, task5],
output = tempdir + "final.6")\
.follows(task3, task4, task5, ) \
.posttask(lambda: do_write(test_file, "Task 6 Done\n"))
def check_job_order_correct(filename):
"""
1 -> 2 -> 3 ->
-> 4 ->
5 -> 6
"""
