def test_newstyle_ruffus (self): test_pipeline = Pipeline("test") test_pipeline.files(create_random_numbers, None, tempdir + "random_numbers.list")\ .follows(mkdir(tempdir)) test_pipeline.split(task_func = step_4_split_numbers_into_chunks, input = tempdir + "random_numbers.list", output = tempdir + "*.chunks")\ .follows(create_random_numbers) test_pipeline.transform(task_func = step_5_calculate_sum_of_squares, input = step_4_split_numbers_into_chunks, filter = suffix(".chunks"), output = ".sums") test_pipeline.merge(task_func = step_6_calculate_variance, input = step_5_calculate_sum_of_squares, output = os.path.join(tempdir, "variance.result"))\ .posttask(lambda: sys.stdout.write(" hooray\n"))\ .posttask(print_hooray_again, print_whoppee_again, touch_file(os.path.join(tempdir, "done"))) test_pipeline.run(multiprocess = 50, verbose = 0) output_file = os.path.join(tempdir, "variance.result") if not os.path.exists (output_file): raise Exception("Missing %s" % output_file)
def test_newstyle_ruffus(self): test_pipeline = Pipeline("test") test_pipeline.files(create_random_numbers, None, tempdir + "random_numbers.list")\ .follows(mkdir(tempdir)) test_pipeline.split(task_func = step_4_split_numbers_into_chunks, input = tempdir + "random_numbers.list", output = tempdir + "*.chunks")\ .follows(create_random_numbers) test_pipeline.transform(task_func=step_5_calculate_sum_of_squares, input=step_4_split_numbers_into_chunks, filter=suffix(".chunks"), output=".sums") test_pipeline.merge(task_func = step_6_calculate_variance, input = step_5_calculate_sum_of_squares, output = os.path.join(tempdir, "variance.result"))\ .posttask(lambda: sys.stdout.write(" hooray\n"))\ .posttask(print_hooray_again, print_whoppee_again, touch_file(os.path.join(tempdir, "done"))) test_pipeline.run(multiprocess=50, verbose=0) output_file = os.path.join(tempdir, "variance.result") if not os.path.exists(output_file): raise Exception("Missing %s" % output_file)
def test_newstyle_task(self): test_pipeline = Pipeline("test") test_pipeline.files(task1, [[None, tempdir + "a.1"], [None, tempdir + "b.1"]])\ .follows(mkdir(tempdir)) test_pipeline.files(task2, [[None, tempdir + "c.1"], [None, tempdir + "d.1"]])\ .follows(mkdir(tempdir)) test_pipeline.transform(task_func=task3, input=task1, filter=regex(r"(.+)"), replace_inputs=ruffus.inputs( ((r"\1"), task2, "test_transform_inputs.*y")), output=r"\1.output") test_pipeline.merge(task4, (task3), tempdir + "final.output") test_pipeline.run([task4], multiprocess=10, verbose=0) correct_output = "{tempdir}a.1.output:test_transform_inputs.py,{tempdir}a.1,{tempdir}c.1,{tempdir}d.1;{tempdir}b.1.output:test_transform_inputs.py,{tempdir}b.1,{tempdir}c.1,{tempdir}d.1;".format( tempdir=tempdir) with open(tempdir + "final.output") as ff: real_output = ff.read() self.assertEqual(correct_output, real_output)
def test_newstyle_task(self): test_pipeline = Pipeline("test") test_pipeline.files(task1, a) save_to_str_logger = t_save_to_str_logger() test_pipeline.run(multiprocess=10, logger=save_to_str_logger, verbose=1) self.assertTrue("@files() was empty" in save_to_str_logger.warning_str) print("\n Warning printed out correctly", file=sys.stderr)
def test_newstyle_task (self): test_pipeline = Pipeline("test") test_pipeline.files(task1, a) save_to_str_logger = t_save_to_str_logger() test_pipeline.run(multiprocess = 10, logger = save_to_str_logger, verbose = 1) self.assertTrue("@files() was empty" in save_to_str_logger.warning_str) print("\n Warning printed out correctly", file=sys.stderr)
def test_newstyle_ruffus (self): test_pipeline = Pipeline("test") test_pipeline.follows(setup_simulation_data, mkdir(gene_data_dir, simulation_data_dir)) test_pipeline.files(gwas_simulation, generate_simulation_params)\ .follows(setup_simulation_data)\ .follows(mkdir(working_dir, os.path.join(working_dir, "simulation_results"))) test_pipeline.collate(statistical_summary, gwas_simulation, regex(r"simulation_results/(\d+).\d+.simulation_res"), r"\1.mean")\ .posttask(lambda : sys.stdout.write("\nOK\n")) test_pipeline.run(multiprocess = 50, verbose = 0) for oo in "000.mean", "001.mean": results_file_name = os.path.join(working_dir, oo) if not os.path.exists(results_file_name): raise Exception("Missing %s" % results_file_name)
def test_newstyle_ruffus(self): test_pipeline = Pipeline("test") test_pipeline.follows(setup_simulation_data, mkdir(gene_data_dir, simulation_data_dir)) test_pipeline.files(gwas_simulation, generate_simulation_params)\ .follows(setup_simulation_data)\ .follows(mkdir(working_dir, os.path.join(working_dir, "simulation_results"))) test_pipeline.collate(statistical_summary, gwas_simulation, regex(r"simulation_results/(\d+).\d+.simulation_res"), r"\1.mean")\ .posttask(lambda: sys.stdout.write("\nOK\n")) test_pipeline.run(multiprocess=50, verbose=0) for oo in "000.mean", "001.mean": results_file_name = os.path.join(working_dir, oo) if not os.path.exists(results_file_name): raise Exception("Missing %s" % results_file_name)
def test_newstyle_task (self): """ Same as above but construct a new pipeline on the fly without decorators """ test_pipeline = Pipeline("test") test_pipeline.files(task1, None, tempdir + 'a.1')\ .follows(mkdir(tempdir)) test_pipeline.transform(task_func = task2, input = task1, filter = regex(r".*"), output = tempdir + 'b.1') test_pipeline.files(task3, task2, tempdir + 'c.1') test_pipeline.files(task4, [[None, tempdir + 'd.1'], [None, tempdir + 'e.1']])\ .follows(task3) test_pipeline.files(task5, task4, tempdir + "f.1") test_pipeline.run(multiprocess = 10, verbose = 0)
def test_newstyle_task(self): """ Same as above but construct a new pipeline on the fly without decorators """ test_pipeline = Pipeline("test") test_pipeline.files(task1, None, tempdir + 'a.1')\ .follows(mkdir(tempdir)) test_pipeline.transform(task_func=task2, input=task1, filter=regex(r".*"), output=tempdir + 'b.1') test_pipeline.files(task3, task2, tempdir + 'c.1') test_pipeline.files(task4, [[None, tempdir + 'd.1'], [None, tempdir + 'e.1']])\ .follows(task3) test_pipeline.files(task5, task4, tempdir + "f.1") test_pipeline.run(multiprocess=10, verbose=0)
input = task1, filter = suffix(".1"), output = ".2") \ .posttask(lambda: do_write(test_file, "Task 2 Done\n")) test_pipeline.transform(task3, task2, regex('(.*).2'), inputs([r"\1.2", tempdir + "a.1"]), r'\1.3')\ .posttask(lambda: do_write(test_file, "Task 3 Done\n")) test_pipeline.transform(task4, tempdir + "*.1", suffix(".1"), ".4")\ .follows(task1)\ .posttask(lambda: do_write(test_file, "Task 4 Done\n"))\ .jobs_limit(1) test_pipeline.files(task5, None, tempdir + 'a.5')\ .follows(mkdir(tempdir))\ .posttask(lambda: do_write(test_file, "Task 5 Done\n")) test_pipeline.merge(task_func = task6, input = [task3, task4, task5], output = tempdir + "final.6")\ .follows(task3, task4, task5, ) \ .posttask(lambda: do_write(test_file, "Task 6 Done\n")) def check_job_order_correct(filename): """ 1 -> 2 -> 3 -> -> 4 -> 5 -> 6 """