def test_newstyle_ruffus(self): test_pipeline = Pipeline("test") test_pipeline.split(task_func=split_fasta_file, input=tempdir + "original.fa", output=[tempdir + "files.split.success", tempdir + "files.split.*.fa"])\ .posttask(lambda: verbose_output.write(" Split into %d files\n" % 10)) test_pipeline.transform(task_func=align_sequences, input=split_fasta_file, filter=suffix(".fa"), output=".aln" # fa -> aln )\ .posttask(lambda: verbose_output.write(" Sequences aligned\n")) test_pipeline.transform(task_func=percentage_identity, input=align_sequences, # find all results from align_sequences # replace suffix with: filter=suffix(".aln"), output=[r".pcid", # .pcid suffix for the result r".pcid_success"] # .pcid_success to indicate job completed )\ .posttask(lambda: verbose_output.write(" %Identity calculated\n")) test_pipeline.merge(task_func=combine_results, input=percentage_identity, output=[tempdir + "all.combine_results", tempdir + "all.combine_results_success"])\ .posttask(lambda: verbose_output.write(" Results recombined\n")) test_pipeline.run(multiprocess=50, verbose=0) if not os.path.exists(tempdir + "all.combine_results"): raise Exception("Missing %s" % (tempdir + "all.combine_results"))
def test_newstyle_ruffus(self): test_pipeline = Pipeline("test") test_pipeline.files(create_random_numbers, None, tempdir + "random_numbers.list")\ .follows(mkdir(tempdir)) test_pipeline.split(task_func = step_4_split_numbers_into_chunks, input = tempdir + "random_numbers.list", output = tempdir + "*.chunks")\ .follows(create_random_numbers) test_pipeline.transform(task_func=step_5_calculate_sum_of_squares, input=step_4_split_numbers_into_chunks, filter=suffix(".chunks"), output=".sums") test_pipeline.merge(task_func = step_6_calculate_variance, input = step_5_calculate_sum_of_squares, output = os.path.join(tempdir, "variance.result"))\ .posttask(lambda: sys.stdout.write(" hooray\n"))\ .posttask(print_hooray_again, print_whoppee_again, touch_file(os.path.join(tempdir, "done"))) test_pipeline.run(multiprocess=50, verbose=0) output_file = os.path.join(tempdir, "variance.result") if not os.path.exists(output_file): raise Exception("Missing %s" % output_file)
def test_newstyle_ruffus(self): # alternative syntax test_pipeline = Pipeline("test") test_pipeline.mkdir(data_dir, work_dir) test_pipeline.originate(task_func=task1, output=[os.path.join(data_dir, "%s.1" % aa) for aa in "abcd"]) test_pipeline.mkdir(filter=suffix(".1"), output=".dir", output_dir=work_dir) test_pipeline.transform(task_func=task2, input=task1, filter=suffix(".1"), output=[".1", ".bak"], extras=["extra.tst", 4, r"orig_dir=\1"], output_dir=work_dir) test_pipeline.subdivide(task3, task2, suffix( ".1"), r"\1.*.2", [r"\1.a.2", r"\1.b.2"], output_dir=data_dir) test_pipeline.transform(task4, task3, suffix( ".2"), ".3", output_dir=work_dir) test_pipeline.merge(task5, task4, os.path.join(data_dir, "summary.5")) test_pipeline.run(multiprocess=50, verbose=0) with open(os.path.join(data_dir, "summary.5")) as ii: active_text = ii.read() if active_text != expected_active_text: raise Exception("Error:\n\tExpected\n%s\nInstead\n%s\n" % (expected_active_text, active_text))
def test_newstyle_task(self): test_pipeline = Pipeline("test") test_pipeline.files(task1, [[None, tempdir + "a.1"], [None, tempdir + "b.1"]])\ .follows(mkdir(tempdir)) test_pipeline.files(task2, [[None, tempdir + "c.1"], [None, tempdir + "d.1"]])\ .follows(mkdir(tempdir)) test_pipeline.transform(task_func=task3, input=task1, filter=regex(r"(.+)"), replace_inputs=ruffus.inputs( ((r"\1"), task2, "test_transform_inputs.*y")), output=r"\1.output") test_pipeline.merge(task4, (task3), tempdir + "final.output") test_pipeline.run([task4], multiprocess=10, verbose=0) correct_output = "{tempdir}a.1.output:test_transform_inputs.py,{tempdir}a.1,{tempdir}c.1,{tempdir}d.1;{tempdir}b.1.output:test_transform_inputs.py,{tempdir}b.1,{tempdir}c.1,{tempdir}d.1;".format( tempdir=tempdir) with open(tempdir + "final.output") as ff: real_output = ff.read() self.assertEqual(correct_output, real_output)
def test_newstyle_ruffus (self): test_pipeline = Pipeline("test") test_pipeline.files(create_random_numbers, None, tempdir + "random_numbers.list")\ .follows(mkdir(tempdir)) test_pipeline.split(task_func = step_4_split_numbers_into_chunks, input = tempdir + "random_numbers.list", output = tempdir + "*.chunks")\ .follows(create_random_numbers) test_pipeline.transform(task_func = step_5_calculate_sum_of_squares, input = step_4_split_numbers_into_chunks, filter = suffix(".chunks"), output = ".sums") test_pipeline.merge(task_func = step_6_calculate_variance, input = step_5_calculate_sum_of_squares, output = os.path.join(tempdir, "variance.result"))\ .posttask(lambda: sys.stdout.write(" hooray\n"))\ .posttask(print_hooray_again, print_whoppee_again, touch_file(os.path.join(tempdir, "done"))) test_pipeline.run(multiprocess = 50, verbose = 0) output_file = os.path.join(tempdir, "variance.result") if not os.path.exists (output_file): raise Exception("Missing %s" % output_file)
def create_pipeline(self): """ Create new pipeline on the fly without using decorators """ global count_pipelines count_pipelines = count_pipelines + 1 test_pipeline = Pipeline("test %d" % count_pipelines) test_pipeline.transform(task_func=transform1, input=input_file, filter=suffix('.txt'), output='.output', extras=[runtime_data]) test_pipeline.transform(task_func=transform_raise_error, input=input_file, filter=suffix('.txt'), output='.output', extras=[runtime_data]) test_pipeline.split(task_func=split1, input=input_file, output=split1_outputs) test_pipeline.merge(task_func=merge2, input=split1, output=merge2_output) return test_pipeline
def test_newstyle_simpler (self): test_pipeline = Pipeline("test") test_pipeline.originate(task1, input_file_names, extras = [logger_proxy, logging_mutex]) test_pipeline.transform(task2, task1, suffix(".1"), ".2", extras = [logger_proxy, logging_mutex]) test_pipeline.transform(task3, task2, suffix(".2"), ".3", extras = [logger_proxy, logging_mutex]) test_pipeline.merge(task4, task3, final_file_name, extras = [logger_proxy, logging_mutex]) #test_pipeline.merge(task4, task3, final_file_name, extras = {"logger_proxy": logger_proxy, "logging_mutex": logging_mutex}) test_pipeline.run(multiprocess = 500, verbose = 0)
def make_pipeline2( pipeline_name = "pipeline2"): test_pipeline2 = Pipeline(pipeline_name) test_pipeline2.transform(task_func = task_1_to_1, # task name name = "44_to_55", # placeholder: will be replaced later with set_input() input = None, filter = suffix(".44"), output = ".55") test_pipeline2.merge( task_func = task_m_to_1, input = test_pipeline2["44_to_55"], output = tempdir + "/final.output",) # Set head and tail test_pipeline2.set_tail_tasks([test_pipeline2[task_m_to_1]]) if not DEBUG_do_not_define_head_task: test_pipeline2.set_head_tasks([test_pipeline2["44_to_55"]]) return test_pipeline2
def test_newstyle_simpler(self): test_pipeline = Pipeline("test") test_pipeline.originate(task1, input_file_names, extras=[logger_proxy, logging_mutex]) test_pipeline.transform(task2, task1, suffix(".1"), ".2", extras=[logger_proxy, logging_mutex]) test_pipeline.transform(task3, task2, suffix(".2"), ".3", extras=[logger_proxy, logging_mutex]) test_pipeline.merge(task4, task3, final_file_name, extras=[logger_proxy, logging_mutex]) #test_pipeline.merge(task4, task3, final_file_name, extras = {"logger_proxy": logger_proxy, "logging_mutex": logging_mutex}) test_pipeline.run(multiprocess=500, verbose=0)
def make_pipeline2(pipeline_name="pipeline2"): test_pipeline2 = Pipeline(pipeline_name) test_pipeline2.transform( task_func=task_1_to_1, # task name name="44_to_55", # placeholder: will be replaced later with set_input() input=None, filter=suffix(".44"), output=".55") test_pipeline2.merge( task_func=task_m_to_1, input=test_pipeline2["44_to_55"], output=tempdir + "/final.output", ) # Set head and tail test_pipeline2.set_tail_tasks([test_pipeline2[task_m_to_1]]) if not DEBUG_do_not_define_head_task: test_pipeline2.set_head_tasks([test_pipeline2["44_to_55"]]) return test_pipeline2
def test_newstyle_ruffus(self): # alternative syntax test_pipeline = Pipeline("test") test_pipeline.mkdir(data_dir, work_dir) test_pipeline.originate( task_func=task1, output=[os.path.join(data_dir, "%s.1" % aa) for aa in "abcd"]) test_pipeline.mkdir(filter=suffix(".1"), output=".dir", output_dir=work_dir) test_pipeline.transform(task_func=task2, input=task1, filter=suffix(".1"), output=[".1", ".bak"], extras=["extra.tst", 4, r"orig_dir=\1"], output_dir=work_dir) test_pipeline.subdivide(task3, task2, suffix(".1"), r"\1.*.2", [r"\1.a.2", r"\1.b.2"], output_dir=data_dir) test_pipeline.transform(task4, task3, suffix(".2"), ".3", output_dir=work_dir) test_pipeline.merge(task5, task4, os.path.join(data_dir, "summary.5")) test_pipeline.run(multiprocess=50, verbose=0) with open(os.path.join(data_dir, "summary.5")) as ii: active_text = ii.read() if active_text != expected_active_text: raise Exception("Error:\n\tExpected\n%s\nInstead\n%s\n" % (expected_active_text, active_text))
def make_pipeline(state): '''Build the pipeline by constructing stages and connecting them together''' # Build an empty pipeline pipeline = Pipeline(name='complexo') # Get a list of paths to all the FASTQ files fastq_files = state.config.get_option('fastqs') # Stages are dependent on the state stages = Stages(state) # The original FASTQ files # This is a dummy stage. It is useful because it makes a node in the # pipeline graph, and gives the pipeline an obvious starting point. pipeline.originate(task_func=stages.original_fastqs, name='original_fastqs', output=fastq_files) # Align paired end reads in FASTQ to the reference producing a BAM file pipeline.transform( task_func=stages.align_bwa, name='align_bwa', input=output_from('original_fastqs'), # Match the R1 (read 1) FASTQ file and grab the path and sample name. # This will be the first input to the stage. # We assume the sample name may consist of only alphanumeric # characters. filter=formatter('.+/(?P<sample>[a-zA-Z0-9_]+)_R1.fastq.gz'), # Add one more inputs to the stage: # 1. The corresponding R2 FASTQ file add_inputs=add_inputs('{path[0]}/{sample[0]}_R2.fastq.gz'), # Add an "extra" argument to the state (beyond the inputs and outputs) # which is the sample name. This is needed within the stage for finding out # sample specific configuration options extras=['{sample[0]}'], # The output file name is the sample name with a .bam extension. output='{path[0]}/{sample[0]}.bam') # Sort the BAM file using Picard pipeline.transform(task_func=stages.sort_bam_picard, name='sort_bam_picard', input=output_from('align_bwa'), filter=suffix('.bam'), output='.sort.bam') # Mark duplicates in the BAM file using Picard pipeline.transform( task_func=stages.mark_duplicates_picard, name='mark_duplicates_picard', input=output_from('sort_bam_picard'), filter=suffix('.sort.bam'), # XXX should make metricsup an extra output? output=['.sort.dedup.bam', '.metricsdup']) # Generate chromosome intervals using GATK pipeline.transform(task_func=stages.chrom_intervals_gatk, name='chrom_intervals_gatk', input=output_from('mark_duplicates_picard'), filter=suffix('.sort.dedup.bam'), output='.chr.intervals') # Local realignment using GATK (pipeline.transform( task_func=stages.local_realignment_gatk, name='local_realignment_gatk', input=output_from('chrom_intervals_gatk'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9_]+).chr.intervals'), add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.bam'), output='{path[0]}/{sample[0]}.sort.dedup.realn.bam').follows( 'mark_duplicates_picard')) # Base recalibration using GATK pipeline.transform(task_func=stages.base_recalibration_gatk, name='base_recalibration_gatk', input=output_from('local_realignment_gatk'), filter=suffix('.sort.dedup.realn.bam'), output=['.recal_data.csv', '.count_cov.log']) # Print reads using GATK (pipeline.transform( task_func=stages.print_reads_gatk, name='print_reads_gatk', input=output_from('base_recalibration_gatk'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9_]+).recal_data.csv'), add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.realn.bam'), output='{path[0]}/{sample[0]}.sort.dedup.realn.recal.bam').follows( 'local_realignment_gatk')) # Call variants using GATK pipeline.transform(task_func=stages.call_variants_gatk, name='call_variants_gatk', input=output_from('print_reads_gatk'), filter=suffix('.sort.dedup.realn.recal.bam'), output='.raw.snps.indels.g.vcf') # Combine G.VCF files for all samples using GATK pipeline.merge(task_func=stages.combine_gvcf_gatk, name='combine_gvcf_gatk', input=output_from('call_variants_gatk'), output='COMPLEXO.mergedgvcf.vcf') # Genotype G.VCF files using GATK pipeline.transform(task_func=stages.genotype_gvcf_gatk, name='genotype_gvcf_gatk', input=output_from('combine_gvcf_gatk'), filter=suffix('.mergedgvcf.vcf'), output='.genotyped.vcf') # SNP recalibration using GATK pipeline.transform(task_func=stages.snp_recalibrate_gatk, name='snp_recalibrate_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.genotyped.vcf'), output=['.snp_recal', '.snp_tranches', '.snp_plots.R']) # INDEL recalibration using GATK pipeline.transform( task_func=stages.indel_recalibrate_gatk, name='indel_recalibrate_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.genotyped.vcf'), output=['.indel_recal', '.indel_tranches', '.indel_plots.R']) # Apply SNP recalibration using GATK (pipeline.transform( task_func=stages.apply_snp_recalibrate_gatk, name='apply_snp_recalibrate_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.genotyped.vcf'), add_inputs=add_inputs(['COMPLEXO.snp_recal', 'COMPLEXO.snp_tranches']), output='.recal_SNP.vcf').follows('snp_recalibrate_gatk')) # Apply INDEL recalibration using GATK (pipeline.transform( task_func=stages.apply_indel_recalibrate_gatk, name='apply_indel_recalibrate_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.genotyped.vcf'), add_inputs=add_inputs( ['COMPLEXO.indel_recal', 'COMPLEXO.indel_tranches']), output='.recal_INDEL.vcf').follows('indel_recalibrate_gatk')) # Combine variants using GATK (pipeline.transform( task_func=stages.combine_variants_gatk, name='combine_variants_gatk', input=output_from('apply_snp_recalibrate_gatk'), filter=suffix('.recal_SNP.vcf'), add_inputs=add_inputs(['COMPLEXO.recal_INDEL.vcf']), output='.combined.vcf').follows('apply_indel_recalibrate_gatk')) # Select variants using GATK pipeline.transform(task_func=stages.select_variants_gatk, name='select_variants_gatk', input=output_from('combine_variants_gatk'), filter=suffix('.combined.vcf'), output='.selected.vcf') return pipeline
test_pipeline.transform(task3, task2, regex('(.*).2'), inputs([r"\1.2", tempdir + "a.1"]), r'\1.3')\ .posttask(lambda: do_write(test_file, "Task 3 Done\n")) test_pipeline.transform(task4, tempdir + "*.1", suffix(".1"), ".4")\ .follows(task1)\ .posttask(lambda: do_write(test_file, "Task 4 Done\n"))\ .jobs_limit(1) test_pipeline.files(task5, None, tempdir + 'a.5')\ .follows(mkdir(tempdir))\ .posttask(lambda: do_write(test_file, "Task 5 Done\n")) test_pipeline.merge(task_func = task6, input = [task3, task4, task5], output = tempdir + "final.6")\ .follows(task3, task4, task5, ) \ .posttask(lambda: do_write(test_file, "Task 6 Done\n")) def check_job_order_correct(filename): """ 1 -> 2 -> 3 -> -> 4 -> 5 -> 6 """ precedence_rules = [[1, 2], [2, 3], [1, 4], [5, 6],
def make_pipeline(state): '''Build the pipeline by constructing stages and connecting them together''' # Build an empty pipeline pipeline = Pipeline(name='cellfree_seq') # Stages are dependent on the state stages = Stages(state) safe_make_dir('alignments') # The original FASTQ files fastq_files = glob.glob('fastqs/*') # This is a dummy stage. It is useful because it makes a node in the # pipeline graph, and gives the pipeline an obvious starting point. pipeline.originate(task_func=stages.original_fastqs, name='original_fastqs', output=fastq_files) # Align paired end reads in FASTQ to the reference producing a BAM file pipeline.transform( task_func=stages.align_bwa, name='align_bwa', input=output_from('original_fastqs'), # Match the R1 (read 1) FASTQ file and grab the path and sample name. # This will be the first input to the stage. filter=formatter('.+/(?P<sample>[a-zA-Z0-9_-]+)_R1.fastq.gz'), # Add one more inputs to the stage: # 1. The corresponding R2 FASTQ file add_inputs=add_inputs('{path[0]}/{sample[0]}_R2.fastq.gz'), # Add an "extra" argument to the state (beyond the inputs and outputs) # which is the sample name. This is needed within the stage for finding out # sample specific configuration options extras=['{sample[0]}'], # The output file name is the sample name with a .bam extension. output='alignments/{sample[0]}.sort.hq.bam') pipeline.transform(task_func=stages.run_connor, name='run_connor', input=output_from('align_bwa'), filter=suffix('.sort.hq.bam'), output='.sort.hq.connor.bam') safe_make_dir('metrics') safe_make_dir('metrics/summary') safe_make_dir('metrics/connor') pipeline.transform( task_func=stages.intersect_bed, name='intersect_bed_raw', input=output_from('align_bwa'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9_-]+).sort.hq.bam'), output='metrics/summary/{sample[0]}.intersectbed.bam') pipeline.transform(task_func=stages.coverage_bed, name='coverage_bed_raw', input=output_from('intersect_bed_raw'), filter=suffix('.intersectbed.bam'), output='.bedtools_hist_all.txt') pipeline.transform( task_func=stages.genome_reads, name='genome_reads_raw', input=output_from('align_bwa'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9_-]+).sort.hq.bam'), output='metrics/summary/{sample[0]}.mapped_to_genome.txt') pipeline.transform(task_func=stages.target_reads, name='target_reads_raw', input=output_from('intersect_bed_raw'), filter=suffix('.intersectbed.bam'), output='.mapped_to_target.txt') pipeline.transform( task_func=stages.total_reads, name='total_reads_raw', input=output_from('align_bwa'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9_-]+).sort.hq.bam'), output='metrics/summary/{sample[0]}.total_raw_reads.txt') pipeline.collate( task_func=stages.generate_stats, name='generate_stats_raw', input=output_from('coverage_bed_raw', 'genome_reads_raw', 'target_reads_raw', 'total_reads_raw'), filter=regex( r'.+/(.+)\.(bedtools_hist_all|mapped_to_genome|mapped_to_target|total_raw_reads)\.txt' ), output=r'metrics/summary/all_sample.summary.\1.txt', extras=[r'\1', 'summary.txt']) pipeline.transform( task_func=stages.intersect_bed, name='intersect_bed_connor', input=output_from('run_connor'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9_-]+).sort.hq.connor.bam'), output='metrics/connor/{sample[0]}.intersectbed.bam') pipeline.transform(task_func=stages.coverage_bed, name='coverage_bed_connor', input=output_from('intersect_bed_connor'), filter=suffix('.intersectbed.bam'), output='.bedtools_hist_all.txt') pipeline.transform( task_func=stages.genome_reads, name='genome_reads_connor', input=output_from('run_connor'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9_-]+).sort.hq.connor.bam'), output='metrics/summary/{sample[0]}.mapped_to_genome.txt') pipeline.transform(task_func=stages.target_reads, name='target_reads_connor', input=output_from('intersect_bed_connor'), filter=suffix('.intersectbed.bam'), output='.mapped_to_target.txt') pipeline.transform( task_func=stages.total_reads, name='total_reads_connor', input=output_from('run_connor'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9_-]+).sort.hq.connor.bam'), output='metrics/summary/{sample[0]}.total_raw_reads.txt') pipeline.collate( task_func=stages.generate_stats, name='generate_stats_connor', input=output_from('coverage_bed_connor', 'genome_reads_connor', 'target_reads_connor', 'total_reads_connor'), filter=regex( r'.+/(.+)\.(bedtools_hist_all|mapped_to_genome|mapped_to_target|total_raw_reads)\.txt' ), output=r'metrics/connor/all_sample.summary.\1.txt', extras=[r'\1', 'connor.summary.txt']) safe_make_dir('variants') safe_make_dir('variants/vardict') pipeline.transform( task_func=stages.run_vardict, name='run_vardict', input=output_from('run_connor'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9_-]+).sort.hq.connor.bam'), output='variants/vardict/{sample[0]}.vcf', extras=['{sample[0]}']) pipeline.transform( task_func=stages.sort_vcfs, name='sort_vcfs', input=output_from('run_vardict'), filter=formatter('variants/vardict/(?P<sample>[a-zA-Z0-9_-]+).vcf'), output='variants/vardict/{sample[0]}.sorted.vcf.gz') pipeline.transform(task_func=stages.index_vcfs, name='index_vcfs', input=output_from('sort_vcfs'), filter=suffix('.sorted.vcf.gz'), output='.sorted.vcf.gz.tbi') (pipeline.merge( task_func=stages.concatenate_vcfs, name='concatenate_vcfs', input=output_from('sort_vcfs'), output='variants/vardict/combined.vcf.gz').follows('index_vcfs')) pipeline.transform(task_func=stages.vt_decompose_normalise, name='vt_decompose_normalise', input=output_from('concatenate_vcfs'), filter=suffix('.vcf.gz'), output='.decomp.norm.vcf.gz') pipeline.transform(task_func=stages.index_vcfs, name='index_final_vcf', input=output_from('vt_decompose_normalise'), filter=suffix('.decomp.norm.vcf.gz'), output='.decomp.norm.vcf.gz.tbi') (pipeline.transform( task_func=stages.apply_vep, name='apply_vep', input=output_from('vt_decompose_normalise'), filter=suffix('.decomp.norm.vcf.gz'), output='.decomp.norm.vep.vcf').follows('index_final_vcf')) return pipeline
def make_pipeline(state): '''Build the pipeline by constructing stages and connecting them together''' # Build an empty pipeline pipeline = Pipeline(name='hiplexpipe') # Get a list of paths to all the FASTQ files fastq_files = state.config.get_option('fastqs') # Stages are dependent on the state stages = Stages(state) # The original FASTQ files # This is a dummy stage. It is useful because it makes a node in the # pipeline graph, and gives the pipeline an obvious starting point. pipeline.originate(task_func=stages.original_fastqs, name='original_fastqs', output=fastq_files) # Align paired end reads in FASTQ to the reference producing a BAM file pipeline.transform( task_func=stages.align_bwa, name='align_bwa', input=output_from('original_fastqs'), # Match the R1 (read 1) FASTQ file and grab the path and sample name. # This will be the first input to the stage. # Hi-Plex example: OHI031002-P02F04_S318_L001_R1_001.fastq # new sample name = OHI031002-P02F04 filter=formatter( '.+/(?P<sample>[a-zA-Z0-9-]+)_(?P<readid>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_R1_(?P<lib>[a-zA-Z0-9-:]+).fastq' ), # Add one more inputs to the stage: # 1. The corresponding R2 FASTQ file # Hi-Plex example: OHI031002-P02F04_S318_L001_R2_001.fastq add_inputs=add_inputs( '{path[0]}/{sample[0]}_{readid[0]}_{lane[0]}_R2_{lib[0]}.fastq'), # Add an "extra" argument to the state (beyond the inputs and outputs) # which is the sample name. This is needed within the stage for finding out # sample specific configuration options extras=['{sample[0]}', '{readid[0]}', '{lane[0]}', '{lib[0]}'], # The output file name is the sample name with a .bam extension. output='alignments/{sample[0]}_{readid[0]}/{sample[0]}_{readid[0]}.bam' ) # Call variants using undr_rover pipeline.transform( task_func=stages.apply_undr_rover, name='apply_undr_rover', input=output_from('original_fastqs'), # Match the R1 (read 1) FASTQ file and grab the path and sample name. # This will be the first input to the stage. filter=formatter( '.+/(?P<sample>[a-zA-Z0-9-]+)_(?P<readid>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_R1_(?P<lib>[a-zA-Z0-9-:]+).fastq' ), add_inputs=add_inputs( '{path[0]}/{sample[0]}_{readid[0]}_{lane[0]}_R2_{lib[0]}.fastq'), # extras=['{sample[0]}', '{readid[0]}', '{lane[0]}', '{lib[0]}'], extras=['{sample[0]}', '{readid[0]}'], # The output file name is the sample name with a .bam extension. output='variants/undr_rover/{sample[0]}_{readid[0]}.vcf') # Sort the BAM file using Picard pipeline.transform(task_func=stages.sort_bam_picard, name='sort_bam_picard', input=output_from('align_bwa'), filter=suffix('.bam'), output='.sort.bam') # High quality and primary alignments pipeline.transform(task_func=stages.primary_bam, name='primary_bam', input=output_from('sort_bam_picard'), filter=suffix('.sort.bam'), output='.primary.bam') # index bam file pipeline.transform(task_func=stages.index_sort_bam_picard, name='index_bam', input=output_from('primary_bam'), filter=suffix('.primary.bam'), output='.primary.bam.bai') # Clip the primer_seq from BAM File (pipeline.transform( task_func=stages.clip_bam, name='clip_bam', input=output_from('primary_bam'), filter=suffix('.primary.bam'), output='.primary.primerclipped.bam').follows('index_bam')) ###### GATK VARIANT CALLING ###### # Call variants using GATK pipeline.transform( task_func=stages.call_haplotypecaller_gatk, name='call_haplotypecaller_gatk', input=output_from('clip_bam'), # filter=suffix('.merged.dedup.realn.bam'), filter=formatter( '.+/(?P<sample>[a-zA-Z0-9-_]+).primary.primerclipped.bam'), output='variants/gatk/{sample[0]}.g.vcf') # .follows('index_sort_bam_picard')) # Combine G.VCF files for all samples using GATK pipeline.merge(task_func=stages.combine_gvcf_gatk, name='combine_gvcf_gatk', input=output_from('call_haplotypecaller_gatk'), output='variants/gatk/ALL.combined.vcf') # Genotype G.VCF files using GATK pipeline.transform(task_func=stages.genotype_gvcf_gatk, name='genotype_gvcf_gatk', input=output_from('combine_gvcf_gatk'), filter=suffix('.combined.vcf'), output='.raw.vcf') # Annotate VCF file using GATK pipeline.transform(task_func=stages.variant_annotator_gatk, name='variant_annotator_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.raw.vcf'), output='.raw.annotate.vcf') # Apply VariantFiltration using GATK pipeline.transform(task_func=stages.apply_variant_filtration_gatk, name='apply_variant_filtration_gatk', input=output_from('variant_annotator_gatk'), filter=suffix('.raw.annotate.vcf'), output='.raw.annotate.filtered.vcf') # Apply NORM (pipeline.transform( task_func=stages.apply_vt, name='apply_vt', input=output_from('apply_variant_filtration_gatk'), filter=suffix('.raw.annotate.filtered.vcf'), # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']), output='.raw.annotate.filtered.norm.vcf').follows( 'apply_variant_filtration_gatk')) # Apply VEP (pipeline.transform( task_func=stages.apply_vep, name='apply_vep', input=output_from('apply_vt'), filter=suffix('.raw.annotate.filtered.norm.vcf'), # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']), output='.raw.annotate.filtered.norm.vep.vcf').follows('apply_vt')) # Apply SnpEff (pipeline.transform( task_func=stages.apply_snpeff, name='apply_snpeff', input=output_from('apply_vep'), filter=suffix('.raw.annotate.filtered.norm.vep.vcf'), # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']), output='.raw.annotate.filtered.norm.vep.snpeff.vcf').follows( 'apply_vep')) # Apply vcfanno (pipeline.transform( task_func=stages.apply_vcfanno, name='apply_vcfanno', input=output_from('apply_snpeff'), filter=suffix('.raw.annotate.filtered.norm.vep.snpeff.vcf'), # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']), output='.annotated.vcf').follows('apply_snpeff')) # Concatenate undr_rover vcf files pipeline.merge(task_func=stages.apply_cat_vcf, name='apply_cat_vcf', input=output_from('apply_undr_rover'), output='variants/undr_rover/ur.vcf.gz') # # Apple VEP on concatenated undr_rover vcf file # (pipeline.transform( # task_func=stages.apply_vep, # name='apply_vep_ur', # input=output_from('apply_cat_vcf'), # filter=suffix('.vcf.gz'), # output='.vep.vcf') # .follows('apply_cat_vcf')) # # # Apply vcfanno on concatenated/vep undr_rover vcf file # (pipeline.transform( # task_func=stages.apply_vcfanno, # name='apply_vcfanno_ur', # input=output_from('apply_vep_ur'), # filter=suffix('.vep.vcf'), # output='.vep.anno.vcf') # .follows('apply_vep_ur')) # # # Apply snpeff # (pipeline.transform( # task_func=stages.apply_snpeff, # name='apply_snpeff_ur', # input=output_from('apply_vcfanno_ur'), # filter=suffix('.vep.anno.vcf'), # output='.vep.anno.snpeff.vcf.gz') # .follows('apply_vcfanno_ur')) # # Apply tabix pipeline.transform(task_func=stages.apply_tabix, name='apply_tabix', input=output_from('apply_cat_vcf'), filter=suffix('.vcf.gz'), output='.vcf.gz.tbi') # # Apply HomopolymerRun # (pipeline.transform( # task_func=stages.apply_homopolymer_ann, # name='apply_homopolymer_ann', # input=output_from('apply_snpeff_ur'), # filter=suffix('.vep.anno.snpeff.vcf.gz'), # output='.annotated.vcf') # .follows('apply_tabix')) # # Apply summarize multi coverage # (pipeline.merge( # task_func=stages.apply_multicov, # name='apply_multicov', # input=output_from('primary_bam'), # # filter=suffix('.primary.bam'), # output='coverage/all.multicov.txt') # .follows('index_bam')) # Apply summarize picard coverage # (pipeline.merge( # task_func=stages.apply_summarize_picard, # name='apply_summarize_picard', # input=output_from('target_coverage'), # output='coverage/all.hsmetrics.txt') # .follows('target_coverage')) # # Apply summarize multicov coverage plots # (pipeline.merge( # task_func=stages.apply_multicov_plots, # name='apply_multicov_plots', # input=output_from('apply_multicov'), # output='coverage/coverage_analysis_main.html') # .follows('apply_multicov')) return pipeline
def make_pipeline(state): '''Build the pipeline by constructing stages and connecting them together''' # Build an empty pipeline pipeline = Pipeline(name='complexo') # Get a list of paths to all the FASTQ files fastq_files = state.config.get_option('fastqs') # Stages are dependent on the state stages = Stages(state) # The original FASTQ files # This is a dummy stage. It is useful because it makes a node in the # pipeline graph, and gives the pipeline an obvious starting point. pipeline.originate( task_func=stages.original_fastqs, name='original_fastqs', output=fastq_files) # Align paired end reads in FASTQ to the reference producing a BAM file pipeline.transform( task_func=stages.align_bwa, name='align_bwa', input=output_from('original_fastqs'), # Match the R1 (read 1) FASTQ file and grab the path and sample name. # This will be the first input to the stage. # We assume the sample name may consist of only alphanumeric # characters. filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+)_R1.fastq.gz'), # Add one more inputs to the stage: # 1. The corresponding R2 FASTQ file add_inputs=add_inputs('{path[0]}/{sample[0]}_R2.fastq.gz'), # Add an "extra" argument to the state (beyond the inputs and outputs) # which is the sample name. This is needed within the stage for finding out # sample specific configuration options extras=['{sample[0]}'], # The output file name is the sample name with a .bam extension. output='{path[0]}/{sample[0]}.bam') # Sort the BAM file using Picard pipeline.transform( task_func=stages.sort_bam_picard, name='sort_bam_picard', input=output_from('align_bwa'), filter=suffix('.bam'), output='.sort.bam') # Mark duplicates in the BAM file using Picard pipeline.transform( task_func=stages.mark_duplicates_picard, name='mark_duplicates_picard', input=output_from('sort_bam_picard'), filter=suffix('.sort.bam'), # XXX should make metricsup an extra output? output=['.sort.dedup.bam', '.metricsdup']) # Generate chromosome intervals using GATK pipeline.transform( task_func=stages.chrom_intervals_gatk, name='chrom_intervals_gatk', input=output_from('mark_duplicates_picard'), filter=suffix('.sort.dedup.bam'), output='.chr.intervals') # Local realignment using GATK (pipeline.transform( task_func=stages.local_realignment_gatk, name='local_realignment_gatk', input=output_from('chrom_intervals_gatk'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).chr.intervals'), add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.bam'), output='{path[0]}/{sample[0]}.sort.dedup.realn.bam') .follows('mark_duplicates_picard')) # Base recalibration using GATK pipeline.transform( task_func=stages.base_recalibration_gatk, name='base_recalibration_gatk', input=output_from('local_realignment_gatk'), filter=suffix('.sort.dedup.realn.bam'), output=['.recal_data.csv', '.count_cov.log']) # Print reads using GATK (pipeline.transform( task_func=stages.print_reads_gatk, name='print_reads_gatk', input=output_from('base_recalibration_gatk'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).recal_data.csv'), add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.realn.bam'), output='{path[0]}/{sample[0]}.sort.dedup.realn.recal.bam') .follows('local_realignment_gatk')) # Call variants using GATK pipeline.transform( task_func=stages.call_variants_gatk, name='call_variants_gatk', input=output_from('print_reads_gatk'), filter=suffix('.sort.dedup.realn.recal.bam'), output='.raw.snps.indels.g.vcf') # Combine G.VCF files for all samples using GATK pipeline.merge( task_func=stages.combine_gvcf_gatk, name='combine_gvcf_gatk', input=output_from('call_variants_gatk'), output='PCExomes.mergegvcf.vcf') # Genotype G.VCF files using GATK pipeline.transform( task_func=stages.genotype_gvcf_gatk, name='genotype_gvcf_gatk', input=output_from('combine_gvcf_gatk'), filter=suffix('.mergegvcf.vcf'), output='.genotyped.vcf') # SNP recalibration using GATK pipeline.transform( task_func=stages.snp_recalibrate_gatk, name='snp_recalibrate_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.genotyped.vcf'), output=['.snp_recal', '.snp_tranches', '.snp_plots.R']) # INDEL recalibration using GATK pipeline.transform( task_func=stages.indel_recalibrate_gatk, name='indel_recalibrate_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.genotyped.vcf'), output=['.indel_recal', '.indel_tranches', '.indel_plots.R']) # Apply SNP recalibration using GATK (pipeline.transform( task_func=stages.apply_snp_recalibrate_gatk, name='apply_snp_recalibrate_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.genotyped.vcf'), add_inputs=add_inputs(['PCExomes.snp_recal', 'PCExomes.snp_tranches']), output='.recal_SNP.vcf') .follows('snp_recalibrate_gatk')) # Apply INDEL recalibration using GATK (pipeline.transform( task_func=stages.apply_indel_recalibrate_gatk, name='apply_indel_recalibrate_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.genotyped.vcf'), add_inputs=add_inputs(['PCExomes.indel_recal', 'PCExomes.indel_tranches']), output='.recal_INDEL.vcf') .follows('indel_recalibrate_gatk')) # Combine variants using GATK (pipeline.transform( task_func=stages.combine_variants_gatk, name='combine_variants_gatk', input=output_from('apply_snp_recalibrate_gatk'), filter=suffix('.recal_SNP.vcf'), add_inputs=add_inputs(['PCExomes.recal_INDEL.vcf']), output='.combined.vcf') .follows('apply_indel_recalibrate_gatk')) # Select variants using GATK pipeline.transform( task_func=stages.select_variants_gatk, name='select_variants_gatk', input=output_from('combine_variants_gatk'), filter=suffix('.combined.vcf'), output='.selected.vcf') return pipeline
def make_pipeline(state): """Build the pipeline by constructing stages and connecting them together""" # Build an empty pipeline pipeline = Pipeline(name="crpipe") # Get a list of paths to all the FASTQ files fastq_files = state.config.get_option("fastqs") # Find the path to the reference genome # Stages are dependent on the state stages = Stages(state) # The original FASTQ files # This is a dummy stage. It is useful because it makes a node in the # pipeline graph, and gives the pipeline an obvious starting point. pipeline.originate(task_func=stages.original_fastqs, name="original_fastqs", output=fastq_files) # Convert FASTQ file to FASTA using fastx toolkit # pipeline.transform( # task_func=stages.fastq_to_fasta, # name='fastq_to_fasta', # input=output_from('original_fastqs'), # filter=suffix('.fastq.gz'), # output='.fasta') # The original reference file # This is a dummy stage. It is useful because it makes a node in the # pipeline graph, and gives the pipeline an obvious starting point. # pipeline.originate( # task_func=stages.original_reference, # name='original_reference', # output=reference_file) # Run fastQC on the FASTQ files pipeline.transform( task_func=stages.fastqc, name="fastqc", input=output_from("original_fastqs"), filter=suffix(".fastq.gz"), output="_fastqc", ) # Index the reference using BWA # pipeline.transform( # task_func=stages.index_reference_bwa, # name='index_reference_bwa', # input=output_from('original_reference'), # filter=suffix('.fa'), # output=['.fa.amb', '.fa.ann', '.fa.pac', '.fa.sa', '.fa.bwt']) # Index the reference using samtools # pipeline.transform( # task_func=stages.index_reference_samtools, # name='index_reference_samtools', # input=output_from('original_reference'), # filter=suffix('.fa'), # output='.fa.fai') # Index the reference using bowtie 2 # pipeline.transform( # task_func=stages.index_reference_bowtie2, # name='index_reference_bowtie2', # input=output_from('original_reference'), # filter=formatter('.+/(?P<refname>[a-zA-Z0-9]+\.fa)'), # output=['{path[0]}/{refname[0]}.1.bt2', # '{path[0]}/{refname[0]}.2.bt2', # '{path[0]}/{refname[0]}.3.bt2', # '{path[0]}/{refname[0]}.4.bt2', # '{path[0]}/{refname[0]}.rev.1.bt2', # '{path[0]}/{refname[0]}.rev.2.bt2'], # extras=['{path[0]}/{refname[0]}']) # # Create a FASTA sequence dictionary for the reference using picard # pipeline.transform( # task_func=stages.reference_dictionary_picard, # name='reference_dictionary_picard', # input=output_from('original_reference'), # filter=suffix('.fa'), # output='.dict') # Align paired end reads in FASTQ to the reference producing a BAM file pipeline.transform( task_func=stages.align_bwa, name="align_bwa", input=output_from("original_fastqs"), # Match the R1 (read 1) FASTQ file and grab the path and sample name. # This will be the first input to the stage. # We assume the sample name may consist of only alphanumeric # characters. filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+)_R1.fastq.gz"), # Add two more inputs to the stage: # 1. The corresponding R2 FASTQ file add_inputs=add_inputs("{path[0]}/{sample[0]}_R2.fastq.gz"), # Add an "extra" argument to the state (beyond the inputs and outputs) # which is the sample name. This is needed within the stage for finding out # sample specific configuration options extras=["{sample[0]}"], # The output file name is the sample name with a .bam extension. output="{path[0]}/{sample[0]}.bam", ) # Sort alignment with sambamba pipeline.transform( task_func=stages.sort_bam_sambamba, name="sort_alignment", input=output_from("align_bwa"), filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).bam"), output="{path[0]}/{sample[0]}.sorted.bam", ) # Extract MMR genes from the sorted BAM file pipeline.transform( task_func=stages.extract_genes_bedtools, name="extract_genes_bedtools", input=output_from("sort_alignment"), filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).sorted.bam"), output="{path[0]}/{sample[0]}.mmr.bam", ) # Extract selected chromosomes from the sorted BAM file pipeline.transform( task_func=stages.extract_chromosomes_samtools, name="extract_chromosomes_samtools", input=output_from("sort_alignment"), filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).sorted.bam"), output="{path[0]}/{sample[0]}.chroms.bam", ) # Index the MMR genes bam file with samtools pipeline.transform( task_func=stages.index_bam, name="index_mmr_alignment", input=output_from("extract_genes_bedtools"), filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).mmr.bam"), output="{path[0]}/{sample[0]}.mmr.bam.bai", ) # Compute depth of coverage of the alignment with GATK DepthOfCoverage # pipeline.transform( # task_func=stages.alignment_coverage_gatk, # name='alignment_coverage_gatk', # input=output_from('sort_alignment'), # filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'), # add_inputs=add_inputs([reference_file]), # output='{path[0]}/{sample[0]}.coverage_summary', # extras=['{path[0]}/{sample[0]}_coverage']) # Index the alignment with samtools pipeline.transform( task_func=stages.index_bam, name="index_alignment", input=output_from("sort_alignment"), filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).sorted.bam"), output="{path[0]}/{sample[0]}.sorted.bam.bai", ) # Generate alignment stats with bamtools pipeline.transform( task_func=stages.bamtools_stats, name="bamtools_stats", input=output_from("align_bwa"), filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).bam"), output="{path[0]}/{sample[0]}.stats.txt", ) # Extract the discordant paired-end alignments pipeline.transform( task_func=stages.extract_discordant_alignments, name="extract_discordant_alignments", input=output_from("align_bwa"), filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).bam"), output="{path[0]}/{sample[0]}.discordants.unsorted.bam", ) # Extract split-read alignments pipeline.transform( task_func=stages.extract_split_read_alignments, name="extract_split_read_alignments", input=output_from("align_bwa"), filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).bam"), output="{path[0]}/{sample[0]}.splitters.unsorted.bam", ) # Sort discordant reads. # Samtools annoyingly takes the prefix of the output bam name as its argument. # So we pass this as an extra argument. However Ruffus needs to know the full name # of the output bam file, so we pass that as the normal output parameter. pipeline.transform( task_func=stages.sort_bam, name="sort_discordants", input=output_from("extract_discordant_alignments"), filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).discordants.unsorted.bam"), extras=["{path[0]}/{sample[0]}.discordants"], output="{path[0]}/{sample[0]}.discordants.bam", ) # Index the sorted discordant bam with samtools # pipeline.transform( # task_func=stages.index_bam, # name='index_discordants', # input=output_from('sort_discordants'), # filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).discordants.bam'), # output='{path[0]}/{sample[0]}.discordants.bam.bai') # Sort discordant reads # Samtools annoyingly takes the prefix of the output bam name as its argument. # So we pass this as an extra argument. However Ruffus needs to know the full name # of the output bam file, so we pass that as the normal output parameter. pipeline.transform( task_func=stages.sort_bam, name="sort_splitters", input=output_from("extract_split_read_alignments"), filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).splitters.unsorted.bam"), extras=["{path[0]}/{sample[0]}.splitters"], output="{path[0]}/{sample[0]}.splitters.bam", ) # Index the sorted splitters bam with samtools # pipeline.transform( # task_func=stages.index_bam, # name='index_splitters', # input=output_from('sort_splitters'), # filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).splitters.bam'), # output='{path[0]}/{sample[0]}.splitters.bam.bai') # Call structural variants with lumpy ( pipeline.transform( task_func=stages.structural_variants_lumpy, name="structural_variants_lumpy", input=output_from("sort_alignment"), filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).sorted.bam"), add_inputs=add_inputs(["{path[0]}/{sample[0]}.splitters.bam", "{path[0]}/{sample[0]}.discordants.bam"]), output="{path[0]}/{sample[0]}.lumpy.vcf", ) .follows("index_alignment") .follows("sort_splitters") .follows("sort_discordants") ) # Call genotypes on lumpy output using SVTyper # (pipeline.transform( # task_func=stages.genotype_svtyper, # name='genotype_svtyper', # input=output_from('structural_variants_lumpy'), # filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).lumpy.vcf'), # add_inputs=add_inputs(['{path[0]}/{sample[0]}.sorted.bam', '{path[0]}/{sample[0]}.splitters.bam']), # output='{path[0]}/{sample[0]}.svtyper.vcf') # .follows('align_bwa') # .follows('sort_splitters') # .follows('index_alignment') # .follows('index_splitters') # .follows('index_discordants')) # Call SVs with Socrates ( pipeline.transform( task_func=stages.structural_variants_socrates, name="structural_variants_socrates", input=output_from("sort_alignment"), filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).sorted.bam"), # output goes to {path[0]}/socrates/ output="{path[0]}/socrates/results_Socrates_paired_{sample[0]}.sorted_long_sc_l25_q5_m5_i95.txt", extras=["{path[0]}"], ) ) # Call DELs with DELLY pipeline.merge( task_func=stages.deletions_delly, name="deletions_delly", input=output_from("sort_alignment"), output="delly.DEL.vcf", ) # Call DUPs with DELLY pipeline.merge( task_func=stages.duplications_delly, name="duplications_delly", input=output_from("sort_alignment"), output="delly.DUP.vcf", ) # Call INVs with DELLY pipeline.merge( task_func=stages.inversions_delly, name="inversions_delly", input=output_from("sort_alignment"), output="delly.INV.vcf", ) # Call TRAs with DELLY pipeline.merge( task_func=stages.translocations_delly, name="translocations_delly", input=output_from("sort_alignment"), output="delly.TRA.vcf", ) # Join both read pair files using gustaf_mate_joining # pipeline.transform( # task_func=stages.gustaf_mate_joining, # name='gustaf_mate_joining', # input=output_from('fastq_to_fasta'), # # Match the R1 (read 1) FASTA file and grab the path and sample name. # # This will be the first input to the stage. # # We assume the sample name may consist of only alphanumeric # # characters. # filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+)_R1.fasta'), # # Add one more input to the stage: # # 1. The corresponding R2 FASTA file # add_inputs=add_inputs(['{path[0]}/{sample[0]}_R2.fasta']), # output='{path[0]}/{sample[0]}.joined_mates.fasta') # Call structural variants with pindel # (pipeline.transform( # task_func=stages.structural_variants_pindel, # name='structural_variants_pindel', # input=output_from('sort_alignment'), # filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'), # add_inputs=add_inputs(['{path[0]}/{sample[0]}.pindel_config.txt', reference_file]), # output='{path[0]}/{sample[0]}.pindel') # .follows('index_reference_bwa') # .follows('index_reference_samtools')) return pipeline
def make_pipeline_process(state): # Define empty pipeline pipeline = Pipeline(name='hiplexpipe') # Get a list of paths to all the directories to be combined for variant calling run_directories = state.config.get_option('runs') #grab files from each of the processed directories in "runs" gatk_files = [] undr_rover_files = [] for directory in run_directories: gatk_files.extend(glob.glob(directory + '/variants/gatk/*.g.vcf')) undr_rover_files.extend( glob.glob(directory + '/variants/undr_rover/*sorted.vcf.gz')) # Stages are dependent on the state stages = Stages(state) # This is a dummy stage. It is useful because it makes a node in the # pipeline graph, and gives the pipeline an obvious starting point. pipeline.originate(task_func=stages.glob_gatk, name='glob_gatk', output=gatk_files) #Dummy stage to grab the undr rover files pipeline.originate(task_func=stages.glob_undr_rover, name='glob_undr_rover', output=undr_rover_files) safe_make_dir('variants') safe_make_dir('variants/gatk') safe_make_dir('variants/undr_rover') pipeline.merge(task_func=stages.concatenate_vcfs, name='concatenate_vcfs', input=output_from('glob_undr_rover'), output='variants/undr_rover/combined_undr_rover.vcf.gz') pipeline.transform(task_func=stages.index_final_vcf, name='index_final_vcf', input=output_from('concatenate_vcfs'), filter=suffix('.vcf.gz'), output='.vcf.gz.tbi') # Combine G.VCF files for all samples using GATK pipeline.merge(task_func=stages.combine_gvcf_gatk, name='combine_gvcf_gatk', input=output_from('glob_gatk'), output='ALL.combined.vcf') # Genotype G.VCF files using GATK pipeline.transform(task_func=stages.genotype_gvcf_gatk, name='genotype_gvcf_gatk', input=output_from('combine_gvcf_gatk'), filter=suffix('.combined.vcf'), output='.raw.vcf') # Apply GT filters to genotyped vcf pipeline.transform(task_func=stages.genotype_filter_gatk, name='genotype_filter_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.raw.vcf'), output='.raw.gt-filter.vcf') # Decompose and normalise multiallelic sites pipeline.transform(task_func=stages.vt_decompose_normalise, name='vt_decompose_normalise', input=output_from('genotype_filter_gatk'), filter=suffix('.raw.gt-filter.vcf'), output='.raw.gt-filter.decomp.norm.vcf') # Annotate VCF file using GATK pipeline.transform(task_func=stages.variant_annotator_gatk, name='variant_annotator_gatk', input=output_from('vt_decompose_normalise'), filter=suffix('.raw.gt-filter.decomp.norm.vcf'), output='.raw.gt-filter.decomp.norm.annotate.vcf') # Filter vcf pipeline.transform( task_func=stages.gatk_filter, name='gatk_filter', input=output_from('variant_annotator_gatk'), filter=suffix('.raw.gt-filter.decomp.norm.annotate.vcf'), output='.raw.gt-filter.decomp.norm.annotate.filter.vcf') #Apply VEP (pipeline.transform( task_func=stages.apply_vep, name='apply_vep', input=output_from('gatk_filter'), filter=suffix('.raw.gt-filter.decomp.norm.annotate.filter.vcf'), add_inputs=add_inputs( ['variants/undr_rover/combined_undr_rover.vcf.gz']), output='.raw.gt-filter.decomp.norm.annotate.filter.vep.vcf').follows( 'index_final_vcf')) return pipeline
def make_pipeline(state): '''Build the pipeline by constructing stages and connecting them together''' # Build an empty pipeline pipeline = Pipeline(name='thepipeline') # Get a list of paths to all the FASTQ files fastq_files = state.config.get_option('fastqs') # Stages are dependent on the state stages = Stages(state) # The original FASTQ files # This is a dummy stage. It is useful because it makes a node in the # pipeline graph, and gives the pipeline an obvious starting point. pipeline.originate( task_func=stages.original_fastqs, name='original_fastqs', output=fastq_files) # Align paired end reads in FASTQ to the reference producing a BAM file pipeline.transform( task_func=stages.align_bwa, name='align_bwa', input=output_from('original_fastqs'), # Match the R1 (read 1) FASTQ file and grab the path and sample name. # This will be the first input to the stage. # We assume the sample name may consist of only alphanumeric # characters. # filter=formatter('(?P<path>.+)/(?P<readid>[a-zA-Z0-9-\.]+)_(?P<lib>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9]+)_1.fastq.gz'), # 1_HFYLVCCXX:2:TCCGCGAA_2_GE0343_1.fastq.gz # 1_HCJWFBCXX:GGACTCCT_L001_9071584415739518822-AGRF-023_R2.fastq.gz filter=formatter( '.+/(?P<readid>[a-zA-Z0-9-]+)_(?P<lib>[a-zA-Z0-9-:]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9-]+)_R1.fastq.gz'), # Add one more inputs to the stage: # 1. The corresponding R2 FASTQ file # e.g. C2WPF.5_Solexa-201237_5_X4311_1.fastq.gz add_inputs=add_inputs( '{path[0]}/{readid[0]}_{lib[0]}_{lane[0]}_{sample[0]}_R2.fastq.gz'), # Add an "extra" argument to the state (beyond the inputs and outputs) # which is the sample name. This is needed within the stage for finding out # sample specific configuration options extras=['{readid[0]}', '{lib[0]}', '{lane[0]}', '{sample[0]}'], # extras=['{sample[0]}'], # The output file name is the sample name with a .bam extension. output='alignments/{sample[0]}/{readid[0]}_{lib[0]}_{lane[0]}_{sample[0]}.bam') # Sort the BAM file using Picard pipeline.transform( task_func=stages.sort_bam_picard, name='sort_bam_picard', input=output_from('align_bwa'), filter=suffix('.bam'), output='.sort.bam') # Mark duplicates in the BAM file using Picard pipeline.transform( task_func=stages.mark_duplicates_picard, name='mark_duplicates_picard', input=output_from('sort_bam_picard'), filter=suffix('.sort.bam'), # XXX should make metricsup an extra output? output=['.sort.dedup.bam', '.metricsdup']) # Local realignment using GATK # Generate RealignerTargetCreator using GATK pipeline.transform( task_func=stages.realigner_target_creator, name='realigner_target_creator', input=output_from('mark_duplicates_picard'), filter=suffix('.sort.dedup.bam'), output='.intervals') # Local realignment using GATK (pipeline.transform( task_func=stages.local_realignment_gatk, name='local_realignment_gatk', input=output_from('realigner_target_creator'), # filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).chr.intervals'), filter=formatter( '.+/(?P<readid>[a-zA-Z0-9-]+)_(?P<lib>[a-zA-Z0-9-:]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9-]+).intervals'), # add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.bam'), add_inputs=add_inputs( 'alignments/{sample[0]}/{readid[0]}_{lib[0]}_{lane[0]}_{sample[0]}.sort.dedup.bam'), output='alignments/{sample[0]}/{readid[0]}_{lib[0]}_{lane[0]}_{sample[0]}.sort.dedup.realn.bam') .follows('mark_duplicates_picard')) # Base recalibration using GATK pipeline.transform( task_func=stages.base_recalibration_gatk, name='base_recalibration_gatk', input=output_from('local_realignment_gatk'), filter=suffix('.sort.dedup.realn.bam'), output=['.recal_data.csv', '.count_cov.log']) # Print reads using GATK (pipeline.transform( task_func=stages.print_reads_gatk, name='print_reads_gatk', input=output_from('base_recalibration_gatk'), # filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).recal_data.csv'), filter=formatter( # '.+/(?P<readid>[a-zA-Z0-9-\.]+)_(?P<lib>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9]+).recal_data.csv'), '.+/(?P<readid>[a-zA-Z0-9-]+)_(?P<lib>[a-zA-Z0-9-:]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9-]+).recal_data.csv'), # '.+/(?P<readid>[a-zA-Z0-9-]+)_(?P<lib>[a-zA-Z0-9-:]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9-]+).recal_data.csv'), # add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.realn.bam'), add_inputs=add_inputs( 'alignments/{sample[0]}/{readid[0]}_{lib[0]}_{lane[0]}_{sample[0]}.sort.dedup.realn.bam'), # output='{path[0]}/{sample[0]}.sort.dedup.realn.recal.bam') output='alignments/{sample[0]}/{readid[0]}_{lib[0]}_{lane[0]}_{sample[0]}.sort.dedup.realn.recal.bam') .follows('local_realignment_gatk')) # Merge lane bams to sample bams pipeline.collate( task_func=stages.merge_sample_bams, name='merge_sample_bams', filter=formatter( # '.+/(?P<readid>[a-zA-Z0-9-\.]+)_(?P<lib>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9]+).sort.dedup.realn.recal.bam'), '.+/(?P<readid>[a-zA-Z0-9-]+)_(?P<lib>[a-zA-Z0-9-:]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9-]+).sort.dedup.realn.recal.bam'), # inputs=add_inputs('alignments/{sample[0]}/{readid[0]}_{lib[0]}_{lane[0]}_{sample[0]}.sort.dedup.realn.bam'), input=output_from('print_reads_gatk'), output='alignments/{sample[0]}/{sample[0]}.merged.bam') # Mark duplicates in the BAM file using Picard pipeline.transform( task_func=stages.mark_duplicates_picard, name='mark_duplicates_picard2', input=output_from('merge_sample_bams'), # filter=formatter( # '.+/(?P<readid>[a-zA-Z0-9-\.]+)_(?P<lib>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9]+).merged.bam'), filter=suffix('.merged.bam'), # XXX should make metricsup an extra output? output=['.merged.dedup.bam', '.metricsdup']) # Local realignment2 using GATK # Generate RealignerTargetCreator using GATK pipeline.transform( task_func=stages.realigner_target_creator, name='realigner_target_creator2', input=output_from('mark_duplicates_picard2'), filter=suffix('.dedup.bam'), output='.intervals') # Local realignment using GATK (pipeline.transform( task_func=stages.local_realignment_gatk, name='local_realignment_gatk2', input=output_from('realigner_target_creator2'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9-]+).merged.intervals'), # filter=formatter( # '.+/(?P<readid>[a-zA-Z0-9-\.]+)_(?P<lib>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9]+).intervals'), # add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.bam'), add_inputs=add_inputs( 'alignments/{sample[0]}/{sample[0]}.merged.dedup.bam'), output='alignments/{sample[0]}/{sample[0]}.merged.dedup.realn.bam') .follows('mark_duplicates_picard2')) # Call variants using GATK pipeline.transform( task_func=stages.call_haplotypecaller_gatk, name='call_haplotypecaller_gatk', input=output_from('local_realignment_gatk2'), # filter=suffix('.merged.dedup.realn.bam'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9-]+).merged.dedup.realn.bam'), output='variants/{sample[0]}.g.vcf') # Combine G.VCF files for all samples using GATK pipeline.merge( task_func=stages.combine_gvcf_gatk, name='combine_gvcf_gatk', input=output_from('call_haplotypecaller_gatk'), output='variants/ALL.combined.vcf') # Genotype G.VCF files using GATK pipeline.transform( task_func=stages.genotype_gvcf_gatk, name='genotype_gvcf_gatk', input=output_from('combine_gvcf_gatk'), filter=suffix('.combined.vcf'), output='.raw.vcf') # SNP recalibration using GATK pipeline.transform( task_func=stages.snp_recalibrate_gatk, name='snp_recalibrate_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.raw.vcf'), output=['.snp_recal', '.snp_tranches', '.snp_plots.R']) # INDEL recalibration using GATK pipeline.transform( task_func=stages.indel_recalibrate_gatk, name='indel_recalibrate_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.raw.vcf'), output=['.indel_recal', '.indel_tranches', '.indel_plots.R']) # Apply SNP recalibration using GATK (pipeline.transform( task_func=stages.apply_snp_recalibrate_gatk, name='apply_snp_recalibrate_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.raw.vcf'), add_inputs=add_inputs(['ALL.snp_recal', 'ALL.snp_tranches']), output='.recal_SNP.vcf') .follows('snp_recalibrate_gatk')) # Apply INDEL recalibration using GATK (pipeline.transform( task_func=stages.apply_indel_recalibrate_gatk, name='apply_indel_recalibrate_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.raw.vcf'), add_inputs=add_inputs( ['ALL.indel_recal', 'ALL.indel_tranches']), output='.recal_INDEL.vcf') .follows('indel_recalibrate_gatk')) # Combine variants using GATK (pipeline.transform( task_func=stages.combine_variants_gatk, name='combine_variants_gatk', input=output_from('apply_snp_recalibrate_gatk'), filter=suffix('.recal_SNP.vcf'), add_inputs=add_inputs(['ALL.recal_INDEL.vcf']), # output='.combined.vcf') output='ALL.raw.vqsr.vcf') .follows('apply_indel_recalibrate_gatk')) # # # Select variants using GATK # pipeline.transform( # task_func=stages.select_variants_gatk, # name='select_variants_gatk', # input=output_from('combine_variants_gatk'), # filter=suffix('.combined.vcf'), # output='.selected.vcf') return pipeline
output=[tempdir + "/g_name.tmp1", tempdir + "/h_name.tmp1"]) test_pipeline1.product(task_func=check_product_task, input=[tempdir + "/" + prefix + "_name.tmp1" for prefix in "abcd"], filter=formatter(".*/(?P<FILE_PART>.+).tmp1$"), input2=generate_initial_files2, filter2=formatter(), input3=generate_initial_files3, filter3=formatter(r"tmp1$"), output="{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.{basename[2][0]}.tmp2", extras=["{basename[0][0][0]}{basename[1][0][0]}{basename[2][0][0]}", # extra: prefices only (abcd etc) # extra: path for 2nd input, 1st file "{subpath[0][0][0]}", "{subdir[0][0][0]}"]).follows("WOWWWEEE").follows(gen_task1).follows(generate_initial_files1).follows("generate_initial_files1") test_pipeline1.merge(task_func=check_product_merged_task, input=check_product_task, output=tempdir + "/merged.results") test_pipeline1.product(task_func=check_product_misspelt_capture_error_task, input=gen_task1, filter=formatter(".*/(?P<FILE_PART>.+).tmp1$"), output="{path[0][0]}/{FILEPART[0][0]}.tmp2") test_pipeline1.product(task_func=check_product_out_of_range_formatter_ref_error_task, input=generate_initial_files1, # filter=formatter(".*/(?P<FILE_PART>.+).tmp1$"), output="{path[2][0]}/{basename[0][0]}.tmp2", extras=["{FILE_PART[0][0]}"]) test_pipeline1.product(task_func=check_product_formatter_ref_index_error_task, input=output_from("generate_initial_files1"), filter=formatter(".*/(?P<FILE_PART>.+).tmp1$"), output="{path[0][0][1000]}/{basename[0][0]}.tmp2", extras=["{FILE_PART[0][0]}"])
def make_pipeline(state): '''Build the pipeline by constructing stages and connecting them together''' # Build an empty pipeline pipeline = Pipeline(name='hiplexpipe') # Get a list of paths to all the FASTQ files fastq_files = state.config.get_option('fastqs') # Stages are dependent on the state stages = Stages(state) # The original FASTQ files # This is a dummy stage. It is useful because it makes a node in the # pipeline graph, and gives the pipeline an obvious starting point. pipeline.originate(task_func=stages.original_fastqs, name='original_fastqs', output=fastq_files) # Align paired end reads in FASTQ to the reference producing a BAM file pipeline.transform( task_func=stages.align_bwa, name='align_bwa', input=output_from('original_fastqs'), # Match the R1 (read 1) FASTQ file and grab the path and sample name. # This will be the first input to the stage. # Hi-Plex example: OHI031002-P02F04_S318_L001_R1_001.fastq # new sample name = OHI031002-P02F04 filter=formatter( '.+/(?P<sample>[a-zA-Z0-9-]+)_(?P<readid>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_R1_(?P<lib>[a-zA-Z0-9-:]+).fastq' ), # Add one more inputs to the stage: # 1. The corresponding R2 FASTQ file # Hi-Plex example: OHI031002-P02F04_S318_L001_R2_001.fastq add_inputs=add_inputs( '{path[0]}/{sample[0]}_{readid[0]}_{lane[0]}_R2_{lib[0]}.fastq'), # Add an "extra" argument to the state (beyond the inputs and outputs) # which is the sample name. This is needed within the stage for finding out # sample specific configuration options extras=['{sample[0]}', '{readid[0]}', '{lane[0]}', '{lib[0]}'], # The output file name is the sample name with a .bam extension. output='alignments/{sample[0]}_{readid[0]}/{sample[0]}_{readid[0]}.bam' ) # Call variants using undr_rover pipeline.transform( task_func=stages.apply_undr_rover, name='apply_undr_rover', input=output_from('original_fastqs'), # Match the R1 (read 1) FASTQ file and grab the path and sample name. # This will be the first input to the stage. filter=formatter( '.+/(?P<sample>[a-zA-Z0-9-]+)_(?P<readid>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_R1_(?P<lib>[a-zA-Z0-9-:]+).fastq' ), add_inputs=add_inputs( '{path[0]}/{sample[0]}_{readid[0]}_{lane[0]}_R2_{lib[0]}.fastq'), # extras=['{sample[0]}', '{readid[0]}', '{lane[0]}', '{lib[0]}'], extras=['{sample[0]}', '{readid[0]}'], # The output file name is the sample name with a .bam extension. output='variants/undr_rover/{sample[0]}_{readid[0]}.vcf') # Sort the BAM file using Picard pipeline.transform(task_func=stages.sort_bam_picard, name='sort_bam_picard', input=output_from('align_bwa'), filter=suffix('.bam'), output='.sort.bam') # High quality and primary alignments pipeline.transform(task_func=stages.primary_bam, name='primary_bam', input=output_from('sort_bam_picard'), filter=suffix('.sort.bam'), output='.primary.bam') # index bam file pipeline.transform(task_func=stages.index_sort_bam_picard, name='index_bam', input=output_from('primary_bam'), filter=suffix('.primary.bam'), output='.primary.bam.bai') # Clip the primer_seq from BAM File (pipeline.transform( task_func=stages.clip_bam, name='clip_bam', input=output_from('primary_bam'), filter=suffix('.primary.bam'), output='.primary.primerclipped.bam').follows('index_bam')) ###### GATK VARIANT CALLING ###### # Call variants using GATK pipeline.transform( task_func=stages.call_haplotypecaller_gatk, name='call_haplotypecaller_gatk', input=output_from('clip_bam'), # filter=suffix('.merged.dedup.realn.bam'), filter=formatter( '.+/(?P<sample>[a-zA-Z0-9-_]+).primary.primerclipped.bam'), output='variants/gatk/{sample[0]}.g.vcf') # .follows('index_sort_bam_picard')) # Combine G.VCF files for all samples using GATK pipeline.merge(task_func=stages.combine_gvcf_gatk, name='combine_gvcf_gatk', input=output_from('call_haplotypecaller_gatk'), output='variants/gatk/ALL.combined.vcf') # Genotype G.VCF files using GATK pipeline.transform(task_func=stages.genotype_gvcf_gatk, name='genotype_gvcf_gatk', input=output_from('combine_gvcf_gatk'), filter=suffix('.combined.vcf'), output='.raw.vcf') # Annotate VCF file using GATK pipeline.transform(task_func=stages.variant_annotator_gatk, name='variant_annotator_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.raw.vcf'), output='.raw.annotate.vcf') # Apply VariantFiltration using GATK pipeline.transform(task_func=stages.apply_variant_filtration_gatk_lenient, name='apply_variant_filtration_gatk_lenient', input=output_from('variant_annotator_gatk'), filter=suffix('.raw.annotate.vcf'), output='.raw.annotate.filtered_lenient.vcf') return pipeline
input=[tempdir + "/" + prefix + "_name.tmp1" for prefix in "abcd"], filter=formatter(".*/(?P<FILE_PART>.+).tmp1$"), input2=generate_initial_files2, filter2=formatter(), input3=generate_initial_files3, filter3=formatter(r"tmp1$"), output= "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.{basename[2][0]}.tmp2", extras=[ "{basename[0][0][0]}{basename[1][0][0]}{basename[2][0][0]}", # extra: prefices only (abcd etc) "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file "{subdir[0][0][0]}" ]).follows("WOWWWEEE").follows(gen_task1).follows( generate_initial_files1).follows("generate_initial_files1") test_pipeline1.merge(task_func=test_product_merged_task, input=test_product_task, output=tempdir + "/merged.results") test_pipeline1.product(task_func=test_product_misspelt_capture_error_task, input=gen_task1, filter=formatter(".*/(?P<FILE_PART>.+).tmp1$"), output="{path[0][0]}/{FILEPART[0][0]}.tmp2") test_pipeline1.product( task_func=test_product_out_of_range_formatter_ref_error_task, input=generate_initial_files1, # filter=formatter(".*/(?P<FILE_PART>.+).tmp1$"), output="{path[2][0]}/{basename[0][0]}.tmp2", extras=["{FILE_PART[0][0]}"]) test_pipeline1.product(task_func=test_product_formatter_ref_index_error_task, input=output_from("generate_initial_files1"), filter=formatter(".*/(?P<FILE_PART>.+).tmp1$"), output="{path[0][0][1000]}/{basename[0][0]}.tmp2",
test_pipeline.transform(task3, task2, regex('(.*).2'), inputs([r"\1.2", tempdir + "a.1"]), r'\1.3')\ .posttask(lambda: do_write(test_file, "Task 3 Done\n")) test_pipeline.transform(task4, tempdir + "*.1", suffix(".1"), ".4")\ .follows(task1)\ .posttask(lambda: do_write(test_file, "Task 4 Done\n"))\ .jobs_limit(1) test_pipeline.files(task5, None, tempdir + 'a.5')\ .follows(mkdir(tempdir))\ .posttask(lambda: do_write(test_file, "Task 5 Done\n")) test_pipeline.merge(task_func = task6, input = [task3, task4, task5], output = tempdir + "final.6")\ .follows(task3, task4, task5, ) \ .posttask(lambda: do_write(test_file, "Task 6 Done\n")) def check_job_order_correct(filename): """ 1 -> 2 -> 3 -> -> 4 -> 5 -> 6 """ precedence_rules = [[1, 2], [2, 3], [1, 4], [5, 6], [3, 6], [4, 6]] index_re = re.compile(r'.*\.([0-9])["\]\n]*$') job_indices = defaultdict(list)
def make_pipeline(state): '''Build the pipeline by constructing stages and connecting them together''' # Build an empty pipeline pipeline = Pipeline(name="radpipe") # Stages are dependent on the state stages = PipelineStages(state) # Get a list of library objects. libraries = parse_libraries( libraries=state.config.get_options("libraries")) # Get a list of input files input_files = [l.files for l in libraries] # input_files = [item for sublist in input_files for item in sublist] state.logger.info("Input files: " + str(input_files)) # Get a list of all samples for each library samples_dict = OrderedDict() for l in libraries: samples_dict[l.name] = l.samples state.logger.debug("Samples: " + str(samples_dict)) # Make sure that there are no duplicate samples sample_list = [ item for sublist in samples_dict.values() for item in sublist ] sample_counts = Counter(sample_list) for sample in sample_counts: if sample_counts[sample] > 1: print("Sample {} appears {} times in the barcodes files. " "Sample names must be unique".format(sample, sample_counts[sample])) sys.exit(radpipe.error_codes.INVALID_INPUT_FILE) # Define output directories output_dir = get_output_paths(state) state.logger.debug(output_dir) # Allow multiple comma-separated tasks if len(state.options.target_tasks) == 1: state.options.target_tasks = state.options.target_tasks[0].split(",") if len(state.options.forced_tasks) == 1: state.options.forced_tasks = state.options.forced_tasks[0].split(",") state.logger.debug("Target tasks: " + str(state.options.target_tasks)) state.logger.debug("Forced tasks: " + str(state.options.forced_tasks)) # Check if alignment_method is valid alignment_method = state.config.get_options( "alignment_method").strip().lower() if alignment_method not in ["bwa mem", "bowtie"]: print("Error: Invalid alignment_method in config file. " \ "Valid options are ['bwa mem', 'bowtie'].") sys.exit(radpipe.error_codes.INVALID_ARGUMENT) if alignment_method == "bwa mem": align_task_name = "bwa_mem" index_task_name = "bwa_index" else: align_task_name = "bowtie" index_task_name = "bowtie_index" # TODO: Refactor this # If 'alignment' is in target_tasks or forced_tasks, specify which # type of alignment job if "alignment" in state.options.target_tasks: index = state.options.target_tasks.index("alignment") state.options.target_tasks[index] = align_task_name if "alignment" in state.options.forced_tasks: index = state.options.forced_tasks.index("alignment") state.options.forced_tasks[index] = align_task_name # If 'build_index' is in target_tasks or forced_tasks, specify which # type of index job if "build_index" in state.options.target_tasks: index = state.options.target_tasks.index("build_index") state.options.target_tasks[index] = index_task_name if "build_index" in state.options.forced_tasks: index = state.options.forced_tasks.index("build_index") state.options.forced_tasks[index] = index_task_name state.logger.debug(state) # Whether to include filter_bam stage or not filter_bams = False try: samtools_view_options = state.config.get_options( "samtools_view_options") if samtools_view_options: filter_bams = True except: pass state.logger.info("Filter bams: {}".format(filter_bams)) # Population map filenames popmap_file = "{output_dir}/{name}_popmap.txt".format( output_dir=output_dir["populations"], name=state.config.get_options("analysis_id")) try: config_popmap_file = state.config.get_options("popmap_file") if config_popmap_file: state.logger.info( "Using popmap file: {}".format(config_popmap_file)) else: raise (Exception) except Exception: config_popmap_file = None state.logger.info("Creating new popmap file: {}".format(popmap_file)) # Population r values populations_r = state.config.get_options("populations_r") assert (isinstance(populations_r, list)) # Dummy stages. These do nothing except provide a node at the beginning # for the pipeline graph, giving the pipeline an obvious starting point. pipeline.originate(task_func=stages.do_nothing, name="original_fastqs", output=input_files) pipeline.originate(task_func=stages.do_nothing, name="reference_genome", output=state.config.get_options("reference_genome")) # Create a copy of the population map file needed for stacks, or create # one denovo using the sample list. pipeline.originate(task_func=stages.create_popmap_file, name="create_popmap_file", output=[popmap_file], extras=[config_popmap_file, sample_list]) # Create index for reference genome based on alignment method. if alignment_method == "bwa mem": pipeline.transform( task_func=stages.bwa_index, name="bwa_index", input=output_from("reference_genome"), filter=formatter(".+/(?P<ref>[^/]+).(fa|fasta)"), output=path_list_join(output_dir["reference"], ["reference.fa.bwt", "reference.fa.sa"]), extras=[output_dir["reference"]]) if alignment_method == "bowtie": pipeline.transform(task_func=stages.bowtie_index, name="bowtie_index", input=output_from("reference_genome"), filter=formatter(".+/(?P<ref>[^/]+).(fa|fasta)"), output=path_list_join( output_dir["reference"], ["reference.1.ebwt", "reference.rev.1.ebwt"]), extras=[output_dir["reference"]]) # FastQC pipeline.transform( task_func=stages.fastqc, name="fastqc", input=output_from("original_fastqs"), filter=formatter(".+/(?P<lib>[^/]+)/(?P<fn>[^/]+).(fastq|fq).gz"), output="%s/{lib[0]}/{fn[0]}_fastqc.zip" % output_dir["fastqc"], extras=[output_dir["fastqc"], "{lib[0]}"]) # MultiQC: FastQC pipeline.merge(task_func=stages.multiqc_fastqc, name="multiqc_fastqc", input=output_from("fastqc"), output="%s/multiqc_fastqc_report.html" % output_dir["qc"], extras=[output_dir["qc"], output_dir["fastqc"]]) # Stacks: Process RAD-Tags pipeline.transform(task_func=stages.process_radtags, name="process_radtags", input=output_from("original_fastqs"), filter=formatter(".+/(?P<lib>[^/]+)/[^/]+"), output="%s/{lib[0]}/{lib[0]}.success" % output_dir["process_radtags"], extras=[ output_dir["process_radtags"], "{lib[0]}", state.config.get_options("renz_1"), state.config.get_options("renz_2"), state.config.get_options("process_radtags_options") ]) # Create a list for alignment with the input fastq files from process_radtags process_radtags_outputs = [] for l in libraries: for s in l.samples: base = "{dir}/{lib}/{sample}".format( dir=output_dir["process_radtags"], lib=l.lib_id, sample=s) process_radtags_outputs.append( [base + ".1.fq.gz", base + ".2.fq.gz"]) # print(process_radtags_outputs) # Alignment if align_task_name == "bwa_mem": (pipeline.transform( task_func=stages.bwa_align, name=align_task_name, input=process_radtags_outputs, filter=formatter(".+/(?P<sm>[^/]+).1.fq.gz"), output="%s/{sm[0]}.bwa.bam" % output_dir["alignments"], extras=[ os.path.join(output_dir["reference"], "reference.fa"), "{path[0]}", output_dir["alignments"], "{sm[0]}", state.config.get_options("alignment_options") ])).follows("bwa_index").follows("process_radtags") if align_task_name == "bowtie": (pipeline.transform( task_func=stages.bowtie_align, name=align_task_name, input=process_radtags_outputs, filter=formatter(".+/(?P<sm>[^/]+).1.fq.gz"), output="%s/{sm[0]}.bowtie.bam" % output_dir["alignments"], extras=[ os.path.join(output_dir["reference"], "reference"), "{path[0]}", output_dir["alignments"], "{sm[0]}", state.config.get_options("alignment_options") ])).follows("bowtie_index").follows("process_radtags") # Sort BAM and index pipeline.transform(task_func=stages.sort_bam, name="sort_bam", input=output_from(align_task_name), filter=suffix(".bam"), output=".sorted.bam") if filter_bams: final_bam_task_name = "filter_bam" pipeline.transform( task_func=stages.filter_bam, name="filter_bam", input=output_from("sort_bam"), filter=suffix(".sorted.bam"), output=".sorted.filtered.bam", extras=[state.config.get_options("samtools_view_options")]) else: final_bam_task_name = "sort_bam" # Samtools flagstat pipeline.transform(task_func=stages.flagstat, name="flagstat", input=output_from(final_bam_task_name), filter=suffix(".bam"), output=".flagstat.txt", output_dir=output_dir["flagstat"]) # MultiQC: flagstat pipeline.merge(task_func=stages.multiqc_flagstat, name="multiqc_flagstat", input=output_from("flagstat"), output="%s/multiqc_flagstat_report.html" % output_dir["qc"], extras=[output_dir["qc"], output_dir["flagstat"]]) # Stacks: gstacks pipeline.merge(task_func=stages.gstacks, name="gstacks", input=output_from(final_bam_task_name), output="%s/catalog.fa.gz" % output_dir["gstacks"], extras=[ output_dir["alignments"], output_dir["gstacks"], align_task_name, final_bam_task_name, sample_list, state.config.get_options("gstacks_options") ]) # Define outputs from each run of populations populations_outputs = [] for r in populations_r: dir_name = "{pop_dir}/{analysis_name}_r{r}".format( pop_dir=output_dir["populations"], analysis_name=state.config.get_options("analysis_id"), r=r) populations_outputs.append( os.path.join(dir_name, "populations.snps.vcf")) # print(populations_outputs) # Stacks: populations pipeline.originate(task_func=stages.populations, name="popluations", output=populations_outputs, extras=[ output_dir["gstacks"], output_dir["populations"], popmap_file, state.config.get_options("populations_options") ]).follows("gstacks").follows("create_popmap_file") return pipeline
null -> "test_active_if/b.1" "test_active_if/b.1" -> "test_active_if/b.2" "test_active_if/b.2" -> "test_active_if/b.4" "test_active_if/b.4" -> "test_active_if/summary.5" """ # alternative syntax test_pipeline = Pipeline("test") test_pipeline.originate(task1, ['test_active_if/a.1', 'test_active_if/b.1'], "an extra_parameter")\ .follows(mkdir("test_active_if")) test_pipeline.transform(task2, task1, suffix(".1"), ".2") test_pipeline.transform(task3, task1, suffix( ".1"), ".3").active_if(lambda: pipeline_active_if) test_pipeline.collate(task4, [task2, task3], regex(r"(.+)\.[23]"), r"\1.4") test_pipeline.merge(task5, task4, "test_active_if/summary.5") class Test_ruffus(unittest.TestCase): def setUp(self): try: shutil.rmtree(tempdir) except: pass os.makedirs(tempdir) def tearDown(self): try: shutil.rmtree(tempdir) pass except:
def make_pipeline(state): '''Build the pipeline by constructing stages and connecting them together''' # Build an empty pipeline pipeline = Pipeline(name='crpipe') # Get a list of paths to all the FASTQ files fastq_files = state.config.get_option('fastqs') # Find the path to the reference genome # Stages are dependent on the state stages = Stages(state) # The original FASTQ files # This is a dummy stage. It is useful because it makes a node in the # pipeline graph, and gives the pipeline an obvious starting point. pipeline.originate(task_func=stages.original_fastqs, name='original_fastqs', output=fastq_files) # Convert FASTQ file to FASTA using fastx toolkit # pipeline.transform( # task_func=stages.fastq_to_fasta, # name='fastq_to_fasta', # input=output_from('original_fastqs'), # filter=suffix('.fastq.gz'), # output='.fasta') # The original reference file # This is a dummy stage. It is useful because it makes a node in the # pipeline graph, and gives the pipeline an obvious starting point. #pipeline.originate( # task_func=stages.original_reference, # name='original_reference', # output=reference_file) # Run fastQC on the FASTQ files pipeline.transform(task_func=stages.fastqc, name='fastqc', input=output_from('original_fastqs'), filter=suffix('.fastq.gz'), output='_fastqc') # Index the reference using BWA #pipeline.transform( # task_func=stages.index_reference_bwa, # name='index_reference_bwa', # input=output_from('original_reference'), # filter=suffix('.fa'), # output=['.fa.amb', '.fa.ann', '.fa.pac', '.fa.sa', '.fa.bwt']) # Index the reference using samtools # pipeline.transform( # task_func=stages.index_reference_samtools, # name='index_reference_samtools', # input=output_from('original_reference'), # filter=suffix('.fa'), # output='.fa.fai') # Index the reference using bowtie 2 # pipeline.transform( # task_func=stages.index_reference_bowtie2, # name='index_reference_bowtie2', # input=output_from('original_reference'), # filter=formatter('.+/(?P<refname>[a-zA-Z0-9]+\.fa)'), # output=['{path[0]}/{refname[0]}.1.bt2', # '{path[0]}/{refname[0]}.2.bt2', # '{path[0]}/{refname[0]}.3.bt2', # '{path[0]}/{refname[0]}.4.bt2', # '{path[0]}/{refname[0]}.rev.1.bt2', # '{path[0]}/{refname[0]}.rev.2.bt2'], # extras=['{path[0]}/{refname[0]}']) # # Create a FASTA sequence dictionary for the reference using picard # pipeline.transform( # task_func=stages.reference_dictionary_picard, # name='reference_dictionary_picard', # input=output_from('original_reference'), # filter=suffix('.fa'), # output='.dict') # Align paired end reads in FASTQ to the reference producing a BAM file pipeline.transform( task_func=stages.align_bwa, name='align_bwa', input=output_from('original_fastqs'), # Match the R1 (read 1) FASTQ file and grab the path and sample name. # This will be the first input to the stage. # We assume the sample name may consist of only alphanumeric # characters. filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+)_R1.fastq.gz'), # Add two more inputs to the stage: # 1. The corresponding R2 FASTQ file add_inputs=add_inputs('{path[0]}/{sample[0]}_R2.fastq.gz'), # Add an "extra" argument to the state (beyond the inputs and outputs) # which is the sample name. This is needed within the stage for finding out # sample specific configuration options extras=['{sample[0]}'], # The output file name is the sample name with a .bam extension. output='{path[0]}/{sample[0]}.bam') # Sort alignment with sambamba pipeline.transform(task_func=stages.sort_bam_sambamba, name='sort_alignment', input=output_from('align_bwa'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).bam'), output='{path[0]}/{sample[0]}.sorted.bam') # Extract MMR genes from the sorted BAM file pipeline.transform( task_func=stages.extract_genes_bedtools, name='extract_genes_bedtools', input=output_from('sort_alignment'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'), output='{path[0]}/{sample[0]}.mmr.bam') # Extract selected chromosomes from the sorted BAM file pipeline.transform( task_func=stages.extract_chromosomes_samtools, name='extract_chromosomes_samtools', input=output_from('sort_alignment'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'), output='{path[0]}/{sample[0]}.chroms.bam') # Index the MMR genes bam file with samtools pipeline.transform(task_func=stages.index_bam, name='index_mmr_alignment', input=output_from('extract_genes_bedtools'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).mmr.bam'), output='{path[0]}/{sample[0]}.mmr.bam.bai') # Compute depth of coverage of the alignment with GATK DepthOfCoverage #pipeline.transform( # task_func=stages.alignment_coverage_gatk, # name='alignment_coverage_gatk', # input=output_from('sort_alignment'), # filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'), # add_inputs=add_inputs([reference_file]), # output='{path[0]}/{sample[0]}.coverage_summary', # extras=['{path[0]}/{sample[0]}_coverage']) # Index the alignment with samtools pipeline.transform( task_func=stages.index_bam, name='index_alignment', input=output_from('sort_alignment'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'), output='{path[0]}/{sample[0]}.sorted.bam.bai') # Generate alignment stats with bamtools pipeline.transform(task_func=stages.bamtools_stats, name='bamtools_stats', input=output_from('align_bwa'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).bam'), output='{path[0]}/{sample[0]}.stats.txt') # Extract the discordant paired-end alignments pipeline.transform(task_func=stages.extract_discordant_alignments, name='extract_discordant_alignments', input=output_from('align_bwa'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).bam'), output='{path[0]}/{sample[0]}.discordants.unsorted.bam') # Extract split-read alignments pipeline.transform(task_func=stages.extract_split_read_alignments, name='extract_split_read_alignments', input=output_from('align_bwa'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).bam'), output='{path[0]}/{sample[0]}.splitters.unsorted.bam') # Sort discordant reads. # Samtools annoyingly takes the prefix of the output bam name as its argument. # So we pass this as an extra argument. However Ruffus needs to know the full name # of the output bam file, so we pass that as the normal output parameter. pipeline.transform( task_func=stages.sort_bam, name='sort_discordants', input=output_from('extract_discordant_alignments'), filter=formatter( '.+/(?P<sample>[a-zA-Z0-9]+).discordants.unsorted.bam'), extras=['{path[0]}/{sample[0]}.discordants'], output='{path[0]}/{sample[0]}.discordants.bam') # Index the sorted discordant bam with samtools # pipeline.transform( # task_func=stages.index_bam, # name='index_discordants', # input=output_from('sort_discordants'), # filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).discordants.bam'), # output='{path[0]}/{sample[0]}.discordants.bam.bai') # Sort discordant reads # Samtools annoyingly takes the prefix of the output bam name as its argument. # So we pass this as an extra argument. However Ruffus needs to know the full name # of the output bam file, so we pass that as the normal output parameter. pipeline.transform( task_func=stages.sort_bam, name='sort_splitters', input=output_from('extract_split_read_alignments'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).splitters.unsorted.bam'), extras=['{path[0]}/{sample[0]}.splitters'], output='{path[0]}/{sample[0]}.splitters.bam') # Index the sorted splitters bam with samtools # pipeline.transform( # task_func=stages.index_bam, # name='index_splitters', # input=output_from('sort_splitters'), # filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).splitters.bam'), # output='{path[0]}/{sample[0]}.splitters.bam.bai') # Call structural variants with lumpy (pipeline.transform( task_func=stages.structural_variants_lumpy, name='structural_variants_lumpy', input=output_from('sort_alignment'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'), add_inputs=add_inputs([ '{path[0]}/{sample[0]}.splitters.bam', '{path[0]}/{sample[0]}.discordants.bam' ]), output='{path[0]}/{sample[0]}.lumpy.vcf').follows('index_alignment'). follows('sort_splitters').follows('sort_discordants')) # Call genotypes on lumpy output using SVTyper #(pipeline.transform( # task_func=stages.genotype_svtyper, # name='genotype_svtyper', # input=output_from('structural_variants_lumpy'), # filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).lumpy.vcf'), # add_inputs=add_inputs(['{path[0]}/{sample[0]}.sorted.bam', '{path[0]}/{sample[0]}.splitters.bam']), # output='{path[0]}/{sample[0]}.svtyper.vcf') # .follows('align_bwa') # .follows('sort_splitters') # .follows('index_alignment') # .follows('index_splitters') # .follows('index_discordants')) # Call SVs with Socrates (pipeline.transform( task_func=stages.structural_variants_socrates, name='structural_variants_socrates', input=output_from('sort_alignment'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'), # output goes to {path[0]}/socrates/ output= '{path[0]}/socrates/results_Socrates_paired_{sample[0]}.sorted_long_sc_l25_q5_m5_i95.txt', extras=['{path[0]}'])) # Call DELs with DELLY pipeline.merge(task_func=stages.deletions_delly, name='deletions_delly', input=output_from('sort_alignment'), output='delly.DEL.vcf') # Call DUPs with DELLY pipeline.merge(task_func=stages.duplications_delly, name='duplications_delly', input=output_from('sort_alignment'), output='delly.DUP.vcf') # Call INVs with DELLY pipeline.merge(task_func=stages.inversions_delly, name='inversions_delly', input=output_from('sort_alignment'), output='delly.INV.vcf') # Call TRAs with DELLY pipeline.merge(task_func=stages.translocations_delly, name='translocations_delly', input=output_from('sort_alignment'), output='delly.TRA.vcf') # Join both read pair files using gustaf_mate_joining #pipeline.transform( # task_func=stages.gustaf_mate_joining, # name='gustaf_mate_joining', # input=output_from('fastq_to_fasta'), # # Match the R1 (read 1) FASTA file and grab the path and sample name. # # This will be the first input to the stage. # # We assume the sample name may consist of only alphanumeric # # characters. # filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+)_R1.fasta'), # # Add one more input to the stage: # # 1. The corresponding R2 FASTA file # add_inputs=add_inputs(['{path[0]}/{sample[0]}_R2.fasta']), # output='{path[0]}/{sample[0]}.joined_mates.fasta') # Call structural variants with pindel #(pipeline.transform( # task_func=stages.structural_variants_pindel, # name='structural_variants_pindel', # input=output_from('sort_alignment'), # filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'), # add_inputs=add_inputs(['{path[0]}/{sample[0]}.pindel_config.txt', reference_file]), # output='{path[0]}/{sample[0]}.pindel') # .follows('index_reference_bwa') # .follows('index_reference_samtools')) return pipeline
def make_pipeline_process(state): #originate process pipeline state # Define empty pipeline pipeline = Pipeline(name='haloplexpipe') # Get a list of paths to all the directories to be combined for variant calling run_directories = state.config.get_option('runs') #grab files from each of the processed directories in "runs" gatk_files = [] for directory in run_directories: gatk_files.extend(glob.glob(directory + '/variants/gatk/*.g.vcf')) stages = Stages(state) #dummy stage to take the globbed outputs of each run that is to be processed pipeline.originate(task_func=stages.glob_gatk, name='glob_gatk', output=gatk_files) # Combine G.VCF files for all samples using GATK pipeline.merge(task_func=stages.combine_gvcf_gatk, name='combine_gvcf_gatk', input=output_from('glob_gatk'), output='processed/gatk/ALL.combined.vcf') # Genotype G.VCF files using GATK pipeline.transform(task_func=stages.genotype_gvcf_gatk, name='genotype_gvcf_gatk', input=output_from('combine_gvcf_gatk'), filter=suffix('.combined.vcf'), output='.raw.vcf') # Apply GT filters to genotyped vcf pipeline.transform(task_func=stages.genotype_filter_gatk, name='genotype_filter_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.raw.vcf'), output='.raw.gt-filter.vcf') # Decompose and normalise multiallelic sites pipeline.transform(task_func=stages.vt_decompose_normalise, name='vt_decompose_normalise', input=output_from('genotype_filter_gatk'), filter=suffix('.raw.gt-filter.vcf'), output='.raw.gt-filter.decomp.norm.vcf') # Annotate VCF file using GATK pipeline.transform(task_func=stages.variant_annotator_gatk, name='variant_annotator_gatk', input=output_from('vt_decompose_normalise'), filter=suffix('.raw.gt-filter.decomp.norm.vcf'), output='.raw.gt-filter.decomp.norm.annotate.vcf') # Filter vcf pipeline.transform( task_func=stages.gatk_filter, name='gatk_filter', input=output_from('variant_annotator_gatk'), filter=suffix('.raw.gt-filter.decomp.norm.annotate.vcf'), output='.raw.gt-filter.decomp.norm.annotate.filter.vcf') #Apply VEP pipeline.transform( task_func=stages.apply_vep, name='apply_vep', input=output_from('gatk_filter'), filter=suffix('.raw.gt-filter.decomp.norm.annotate.filter.vcf'), output='.raw.gt-filter.decomp.norm.annotate.filter.vep.vcf') ####### vardict stuff vardict_files = [] for directory in run_directories: vardict_files.extend( glob.glob(directory + '/variants/vardict/*sorted.vcf.gz')) #dummy stage to take the globbed outputs of each run that is to be processed pipeline.originate(task_func=stages.glob_vardict, name='glob_vardict', output=vardict_files) safe_make_dir('processed/vardict') #concatenate all vardict vcfs pipeline.merge(task_func=stages.concatenate_vcfs, name='concatenate_vcfs', input=output_from('glob_vardict'), output='processed/vardict/combined.vcf.gz') pipeline.transform(task_func=stages.vt_decompose_normalise, name='vt_decompose_normalise_vardict', input=output_from('concatenate_vcfs'), filter=suffix('.vcf.gz'), output='.decomp.norm.vcf.gz') pipeline.transform(task_func=stages.index_vcfs, name='index_final_vcf', input=output_from('vt_decompose_normalise_vardict'), filter=suffix('.decomp.norm.vcf.gz'), output='.decomp.norm.vcf.gz.tbi') (pipeline.transform( task_func=stages.apply_vep, name='apply_vep_vardict', input=output_from('vt_decompose_normalise_vardict'), filter=suffix('.decomp.norm.vcf.gz'), output='.decomp.norm.vep.vcf').follows('index_final_vcf')) return pipeline
"test_active_if/a.2" -> "test_active_if/a.4" null -> "test_active_if/b.1" "test_active_if/b.1" -> "test_active_if/b.2" "test_active_if/b.2" -> "test_active_if/b.4" "test_active_if/b.4" -> "test_active_if/summary.5" """ # alternative syntax test_pipeline = Pipeline("test") test_pipeline.originate(task1, ['test_active_if/a.1', 'test_active_if/b.1'], "an extra_parameter")\ .follows(mkdir("test_active_if")) test_pipeline.transform(task2, task1, suffix(".1"), ".2") test_pipeline.transform(task3, task1, suffix(".1"), ".3").active_if(lambda: pipeline_active_if) test_pipeline.collate(task4, [task2, task3], regex(r"(.+)\.[23]"), r"\1.4") test_pipeline.merge(task5, task4, "test_active_if/summary.5") class Test_ruffus(unittest.TestCase): def setUp(self): try: shutil.rmtree(tempdir) except: pass os.makedirs(tempdir) def tearDown(self): try: shutil.rmtree(tempdir) pass except:
def make_pipeline(state): '''Build the pipeline by constructing stages and connecting them together''' # Build an empty pipeline pipeline = Pipeline(name='vcf_annotation') # Get a list of paths to all the FASTQ files vcf_files = state.config.get_option('vcfs') # Stages are dependent on the state stages = Stages(state) # The original VCF files # This is a dummy stage. It is useful because it makes a node in the # pipeline graph, and gives the pipeline an obvious starting point. pipeline.originate( task_func=stages.original_vcf, name='original_vcf', output=vcf_file) # Decompose VCF using Vt pipeline.transform( task_func=stages.decompose_vcf, name='decompose_vcf', input=output_from('original_vcf'), # This will be the first input to the stage. # We assume the sample name may consist of only alphanumeric # characters. filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).vcf'), # Add an "extra" argument to the state (beyond the inputs and outputs) # which is the VCF file name (e.g. study/family name. # This is needed within the stage for finding out sample specific # configuration options extras=['{sample[0]}'], # The output file name is the sample name with a .bam extension. output='{path[0]}/{sample[0]}.decompose.normalize.vcf') # FILTER COMMON VARIANTS # ADD FILTER COMMON VARIANTS USING VEP # Annotate using VEP pipeline.transform( task_func=stages.annotate_vep, name='annotate_vep', input=output_from('decompose_vcf'), filter=suffix('.vcf'), output='.vep.vcf') # Annotate using SnpEff pipeline.transform( task_func=stages.annotate_snpeff, name='annotate_snpeff', input=output_from('annotate_vep'), filter=suffix('.vcf'), output='.snpeff.vcf') # Mark duplicates in the BAM file using Picard pipeline.transform( task_func=stages.mark_duplicates_picard, name='mark_duplicates_picard', input=output_from('sort_bam_picard'), filter=suffix('.sort.bam'), # XXX should make metricsup an extra output? output=['.sort.dedup.bam', '.metricsdup']) # Generate chromosome intervals using GATK pipeline.transform( task_func=stages.chrom_intervals_gatk, name='chrom_intervals_gatk', input=output_from('mark_duplicates_picard'), filter=suffix('.sort.dedup.bam'), output='.chr.intervals') # Local realignment using GATK (pipeline.transform( task_func=stages.local_realignment_gatk, name='local_realignment_gatk', input=output_from('chrom_intervals_gatk'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).chr.intervals'), add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.bam'), output='{path[0]}/{sample[0]}.sort.dedup.realn.bam') .follows('mark_duplicates_picard')) # Base recalibration using GATK pipeline.transform( task_func=stages.base_recalibration_gatk, name='base_recalibration_gatk', input=output_from('local_realignment_gatk'), filter=suffix('.sort.dedup.realn.bam'), output=['.recal_data.csv', '.count_cov.log']) # Print reads using GATK (pipeline.transform( task_func=stages.print_reads_gatk, name='print_reads_gatk', input=output_from('base_recalibration_gatk'), filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).recal_data.csv'), add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.realn.bam'), output='{path[0]}/{sample[0]}.sort.dedup.realn.recal.bam') .follows('local_realignment_gatk')) # Call variants using GATK pipeline.transform( task_func=stages.call_variants_gatk, name='call_variants_gatk', input=output_from('print_reads_gatk'), filter=suffix('.sort.dedup.realn.recal.bam'), output='.raw.snps.indels.g.vcf') # Combine G.VCF files for all samples using GATK pipeline.merge( task_func=stages.combine_gvcf_gatk, name='combine_gvcf_gatk', input=output_from('call_variants_gatk'), output='PCExomes.mergegvcf.vcf') # Genotype G.VCF files using GATK pipeline.transform( task_func=stages.genotype_gvcf_gatk, name='genotype_gvcf_gatk', input=output_from('combine_gvcf_gatk'), filter=suffix('.mergegvcf.vcf'), output='.genotyped.vcf') # SNP recalibration using GATK pipeline.transform( task_func=stages.snp_recalibrate_gatk, name='snp_recalibrate_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.genotyped.vcf'), output=['.snp_recal', '.snp_tranches', '.snp_plots.R']) # INDEL recalibration using GATK pipeline.transform( task_func=stages.indel_recalibrate_gatk, name='indel_recalibrate_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.genotyped.vcf'), output=['.indel_recal', '.indel_tranches', '.indel_plots.R']) # Apply SNP recalibration using GATK (pipeline.transform( task_func=stages.apply_snp_recalibrate_gatk, name='apply_snp_recalibrate_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.genotyped.vcf'), add_inputs=add_inputs(['PCExomes.snp_recal', 'PCExomes.snp_tranches']), output='.recal_SNP.vcf') .follows('snp_recalibrate_gatk')) # Apply INDEL recalibration using GATK (pipeline.transform( task_func=stages.apply_indel_recalibrate_gatk, name='apply_indel_recalibrate_gatk', input=output_from('genotype_gvcf_gatk'), filter=suffix('.genotyped.vcf'), add_inputs=add_inputs( ['PCExomes.indel_recal', 'PCExomes.indel_tranches']), output='.recal_INDEL.vcf') .follows('indel_recalibrate_gatk')) # Combine variants using GATK (pipeline.transform( task_func=stages.combine_variants_gatk, name='combine_variants_gatk', input=output_from('apply_snp_recalibrate_gatk'), filter=suffix('.recal_SNP.vcf'), add_inputs=add_inputs(['PCExomes.recal_INDEL.vcf']), output='.combined.vcf') .follows('apply_indel_recalibrate_gatk')) # Select variants using GATK pipeline.transform( task_func=stages.select_variants_gatk, name='select_variants_gatk', input=output_from('combine_variants_gatk'), filter=suffix('.combined.vcf'), output='.selected.vcf') return pipeline