def test_newstyle_ruffus(self):
test_pipeline = Pipeline("test")
test_pipeline.split(task_func=split_fasta_file,
input=tempdir + "original.fa",
output=[tempdir + "files.split.success",
tempdir + "files.split.*.fa"])\
.posttask(lambda: verbose_output.write(" Split into %d files\n" % 10))
test_pipeline.transform(task_func=align_sequences,
input=split_fasta_file,
filter=suffix(".fa"),
output=".aln" # fa -> aln
)\
.posttask(lambda: verbose_output.write(" Sequences aligned\n"))
test_pipeline.transform(task_func=percentage_identity,
input=align_sequences, # find all results from align_sequences
# replace suffix with:
filter=suffix(".aln"),
output=[r".pcid", # .pcid suffix for the result
r".pcid_success"] # .pcid_success to indicate job completed
)\
.posttask(lambda: verbose_output.write(" %Identity calculated\n"))
test_pipeline.merge(task_func=combine_results,
input=percentage_identity,
output=[tempdir + "all.combine_results",
tempdir + "all.combine_results_success"])\
.posttask(lambda: verbose_output.write(" Results recombined\n"))
test_pipeline.run(multiprocess=50, verbose=0)
if not os.path.exists(tempdir + "all.combine_results"):
raise Exception("Missing %s" % (tempdir + "all.combine_results"))
def test_newstyle_ruffus(self):
test_pipeline = Pipeline("test")
test_pipeline.files(create_random_numbers, None, tempdir + "random_numbers.list")\
.follows(mkdir(tempdir))
test_pipeline.split(task_func = step_4_split_numbers_into_chunks,
input = tempdir + "random_numbers.list",
output = tempdir + "*.chunks")\
.follows(create_random_numbers)
test_pipeline.transform(task_func=step_5_calculate_sum_of_squares,
input=step_4_split_numbers_into_chunks,
filter=suffix(".chunks"),
output=".sums")
test_pipeline.merge(task_func = step_6_calculate_variance, input = step_5_calculate_sum_of_squares, output = os.path.join(tempdir, "variance.result"))\
.posttask(lambda: sys.stdout.write(" hooray\n"))\
.posttask(print_hooray_again, print_whoppee_again, touch_file(os.path.join(tempdir, "done")))
test_pipeline.run(multiprocess=50, verbose=0)
output_file = os.path.join(tempdir, "variance.result")
if not os.path.exists(output_file):
raise Exception("Missing %s" % output_file)
def test_newstyle_ruffus(self):
# alternative syntax
test_pipeline = Pipeline("test")
test_pipeline.mkdir(data_dir, work_dir)
test_pipeline.originate(task_func=task1,
output=[os.path.join(data_dir, "%s.1" % aa) for aa in "abcd"])
test_pipeline.mkdir(filter=suffix(".1"),
output=".dir",
output_dir=work_dir)
test_pipeline.transform(task_func=task2,
input=task1,
filter=suffix(".1"),
output=[".1", ".bak"],
extras=["extra.tst", 4, r"orig_dir=\1"],
output_dir=work_dir)
test_pipeline.subdivide(task3, task2, suffix(
".1"), r"\1.*.2", [r"\1.a.2", r"\1.b.2"], output_dir=data_dir)
test_pipeline.transform(task4, task3, suffix(
".2"), ".3", output_dir=work_dir)
test_pipeline.merge(task5, task4, os.path.join(data_dir, "summary.5"))
test_pipeline.run(multiprocess=50, verbose=0)
with open(os.path.join(data_dir, "summary.5")) as ii:
active_text = ii.read()
if active_text != expected_active_text:
raise Exception("Error:\n\tExpected\n%s\nInstead\n%s\n" %
(expected_active_text, active_text))
def test_newstyle_task(self):
test_pipeline = Pipeline("test")
test_pipeline.files(task1, [[None, tempdir + "a.1"], [None, tempdir + "b.1"]])\
.follows(mkdir(tempdir))
test_pipeline.files(task2, [[None, tempdir + "c.1"], [None, tempdir + "d.1"]])\
.follows(mkdir(tempdir))
test_pipeline.transform(task_func=task3,
input=task1,
filter=regex(r"(.+)"),
replace_inputs=ruffus.inputs(
((r"\1"), task2,
"test_transform_inputs.*y")),
output=r"\1.output")
test_pipeline.merge(task4, (task3), tempdir + "final.output")
test_pipeline.run([task4], multiprocess=10, verbose=0)
correct_output = "{tempdir}a.1.output:test_transform_inputs.py,{tempdir}a.1,{tempdir}c.1,{tempdir}d.1;{tempdir}b.1.output:test_transform_inputs.py,{tempdir}b.1,{tempdir}c.1,{tempdir}d.1;".format(
tempdir=tempdir)
with open(tempdir + "final.output") as ff:
real_output = ff.read()
self.assertEqual(correct_output, real_output)
def test_newstyle_ruffus (self):
test_pipeline = Pipeline("test")
test_pipeline.files(create_random_numbers, None, tempdir + "random_numbers.list")\
.follows(mkdir(tempdir))
test_pipeline.split(task_func = step_4_split_numbers_into_chunks,
input = tempdir + "random_numbers.list",
output = tempdir + "*.chunks")\
.follows(create_random_numbers)
test_pipeline.transform(task_func = step_5_calculate_sum_of_squares,
input = step_4_split_numbers_into_chunks,
filter = suffix(".chunks"),
output = ".sums")
test_pipeline.merge(task_func = step_6_calculate_variance, input = step_5_calculate_sum_of_squares, output = os.path.join(tempdir, "variance.result"))\
.posttask(lambda: sys.stdout.write(" hooray\n"))\
.posttask(print_hooray_again, print_whoppee_again, touch_file(os.path.join(tempdir, "done")))
test_pipeline.run(multiprocess = 50, verbose = 0)
output_file = os.path.join(tempdir, "variance.result")
if not os.path.exists (output_file):
raise Exception("Missing %s" % output_file)
def create_pipeline(self):
"""
Create new pipeline on the fly without using decorators
"""
global count_pipelines
count_pipelines = count_pipelines + 1
test_pipeline = Pipeline("test %d" % count_pipelines)
test_pipeline.transform(task_func=transform1,
input=input_file,
filter=suffix('.txt'),
output='.output',
extras=[runtime_data])
test_pipeline.transform(task_func=transform_raise_error,
input=input_file,
filter=suffix('.txt'),
output='.output',
extras=[runtime_data])
test_pipeline.split(task_func=split1,
input=input_file,
output=split1_outputs)
test_pipeline.merge(task_func=merge2,
input=split1,
output=merge2_output)
return test_pipeline
def create_pipeline(self):
"""
Create new pipeline on the fly without using decorators
"""
global count_pipelines
count_pipelines = count_pipelines + 1
test_pipeline = Pipeline("test %d" % count_pipelines)
test_pipeline.transform(task_func=transform1,
input=input_file,
filter=suffix('.txt'),
output='.output',
extras=[runtime_data])
test_pipeline.transform(task_func=transform_raise_error,
input=input_file,
filter=suffix('.txt'),
output='.output',
extras=[runtime_data])
test_pipeline.split(task_func=split1,
input=input_file,
output=split1_outputs)
test_pipeline.merge(task_func=merge2,
input=split1,
output=merge2_output)
return test_pipeline
def test_newstyle_simpler (self):
test_pipeline = Pipeline("test")
test_pipeline.originate(task1, input_file_names, extras = [logger_proxy, logging_mutex])
test_pipeline.transform(task2, task1, suffix(".1"), ".2", extras = [logger_proxy, logging_mutex])
test_pipeline.transform(task3, task2, suffix(".2"), ".3", extras = [logger_proxy, logging_mutex])
test_pipeline.merge(task4, task3, final_file_name, extras = [logger_proxy, logging_mutex])
#test_pipeline.merge(task4, task3, final_file_name, extras = {"logger_proxy": logger_proxy, "logging_mutex": logging_mutex})
test_pipeline.run(multiprocess = 500, verbose = 0)
def make_pipeline2( pipeline_name = "pipeline2"):
test_pipeline2 = Pipeline(pipeline_name)
test_pipeline2.transform(task_func = task_1_to_1,
# task name
name = "44_to_55",
# placeholder: will be replaced later with set_input()
input = None,
filter = suffix(".44"),
output = ".55")
test_pipeline2.merge( task_func = task_m_to_1,
input = test_pipeline2["44_to_55"],
output = tempdir + "/final.output",)
# Set head and tail
test_pipeline2.set_tail_tasks([test_pipeline2[task_m_to_1]])
if not DEBUG_do_not_define_head_task:
test_pipeline2.set_head_tasks([test_pipeline2["44_to_55"]])
return test_pipeline2
def test_newstyle_simpler(self):
test_pipeline = Pipeline("test")
test_pipeline.originate(task1,
input_file_names,
extras=[logger_proxy, logging_mutex])
test_pipeline.transform(task2,
task1,
suffix(".1"),
".2",
extras=[logger_proxy, logging_mutex])
test_pipeline.transform(task3,
task2,
suffix(".2"),
".3",
extras=[logger_proxy, logging_mutex])
test_pipeline.merge(task4,
task3,
final_file_name,
extras=[logger_proxy, logging_mutex])
#test_pipeline.merge(task4, task3, final_file_name, extras = {"logger_proxy": logger_proxy, "logging_mutex": logging_mutex})
test_pipeline.run(multiprocess=500, verbose=0)
def make_pipeline2(pipeline_name="pipeline2"):
test_pipeline2 = Pipeline(pipeline_name)
test_pipeline2.transform(
task_func=task_1_to_1,
# task name
name="44_to_55",
# placeholder: will be replaced later with set_input()
input=None,
filter=suffix(".44"),
output=".55")
test_pipeline2.merge(
task_func=task_m_to_1,
input=test_pipeline2["44_to_55"],
output=tempdir + "/final.output",
)
# Set head and tail
test_pipeline2.set_tail_tasks([test_pipeline2[task_m_to_1]])
if not DEBUG_do_not_define_head_task:
test_pipeline2.set_head_tasks([test_pipeline2["44_to_55"]])
return test_pipeline2
def test_newstyle_ruffus(self):
# alternative syntax
test_pipeline = Pipeline("test")
test_pipeline.mkdir(data_dir, work_dir)
test_pipeline.originate(
task_func=task1,
output=[os.path.join(data_dir, "%s.1" % aa) for aa in "abcd"])
test_pipeline.mkdir(filter=suffix(".1"),
output=".dir",
output_dir=work_dir)
test_pipeline.transform(task_func=task2,
input=task1,
filter=suffix(".1"),
output=[".1", ".bak"],
extras=["extra.tst", 4, r"orig_dir=\1"],
output_dir=work_dir)
test_pipeline.subdivide(task3,
task2,
suffix(".1"),
r"\1.*.2", [r"\1.a.2", r"\1.b.2"],
output_dir=data_dir)
test_pipeline.transform(task4,
task3,
suffix(".2"),
".3",
output_dir=work_dir)
test_pipeline.merge(task5, task4, os.path.join(data_dir, "summary.5"))
test_pipeline.run(multiprocess=50, verbose=0)
with open(os.path.join(data_dir, "summary.5")) as ii:
active_text = ii.read()
if active_text != expected_active_text:
raise Exception("Error:\n\tExpected\n%s\nInstead\n%s\n" %
(expected_active_text, active_text))
def test_newstyle_task(self):
test_pipeline = Pipeline("test")
test_pipeline.files(task1, [[None, tempdir + "a.1"], [None, tempdir + "b.1"]])\
.follows(mkdir(tempdir))
test_pipeline.files(task2, [[None, tempdir + "c.1"], [None, tempdir + "d.1"]])\
.follows(mkdir(tempdir))
test_pipeline.transform(task_func=task3,
input=task1,
filter=regex(r"(.+)"),
replace_inputs=ruffus.inputs(
((r"\1"), task2, "test_transform_inputs.*y")),
output=r"\1.output")
test_pipeline.merge(task4, (task3), tempdir + "final.output")
test_pipeline.run([task4], multiprocess=10, verbose=0)
correct_output = "{tempdir}a.1.output:test_transform_inputs.py,{tempdir}a.1,{tempdir}c.1,{tempdir}d.1;{tempdir}b.1.output:test_transform_inputs.py,{tempdir}b.1,{tempdir}c.1,{tempdir}d.1;".format(
tempdir=tempdir)
with open(tempdir + "final.output") as ff:
real_output = ff.read()
self.assertEqual(correct_output, real_output)
def make_pipeline(state):
'''Build the pipeline by constructing stages and connecting them together'''
# Build an empty pipeline
pipeline = Pipeline(name='complexo')
# Get a list of paths to all the FASTQ files
fastq_files = state.config.get_option('fastqs')
# Stages are dependent on the state
stages = Stages(state)
# The original FASTQ files
# This is a dummy stage. It is useful because it makes a node in the
# pipeline graph, and gives the pipeline an obvious starting point.
pipeline.originate(task_func=stages.original_fastqs,
name='original_fastqs',
output=fastq_files)
# Align paired end reads in FASTQ to the reference producing a BAM file
pipeline.transform(
task_func=stages.align_bwa,
name='align_bwa',
input=output_from('original_fastqs'),
# Match the R1 (read 1) FASTQ file and grab the path and sample name.
# This will be the first input to the stage.
# We assume the sample name may consist of only alphanumeric
# characters.
filter=formatter('.+/(?P<sample>[a-zA-Z0-9_]+)_R1.fastq.gz'),
# Add one more inputs to the stage:
# 1. The corresponding R2 FASTQ file
add_inputs=add_inputs('{path[0]}/{sample[0]}_R2.fastq.gz'),
# Add an "extra" argument to the state (beyond the inputs and outputs)
# which is the sample name. This is needed within the stage for finding out
# sample specific configuration options
extras=['{sample[0]}'],
# The output file name is the sample name with a .bam extension.
output='{path[0]}/{sample[0]}.bam')
# Sort the BAM file using Picard
pipeline.transform(task_func=stages.sort_bam_picard,
name='sort_bam_picard',
input=output_from('align_bwa'),
filter=suffix('.bam'),
output='.sort.bam')
# Mark duplicates in the BAM file using Picard
pipeline.transform(
task_func=stages.mark_duplicates_picard,
name='mark_duplicates_picard',
input=output_from('sort_bam_picard'),
filter=suffix('.sort.bam'),
# XXX should make metricsup an extra output?
output=['.sort.dedup.bam', '.metricsdup'])
# Generate chromosome intervals using GATK
pipeline.transform(task_func=stages.chrom_intervals_gatk,
name='chrom_intervals_gatk',
input=output_from('mark_duplicates_picard'),
filter=suffix('.sort.dedup.bam'),
output='.chr.intervals')
# Local realignment using GATK
(pipeline.transform(
task_func=stages.local_realignment_gatk,
name='local_realignment_gatk',
input=output_from('chrom_intervals_gatk'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9_]+).chr.intervals'),
add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.bam'),
output='{path[0]}/{sample[0]}.sort.dedup.realn.bam').follows(
'mark_duplicates_picard'))
# Base recalibration using GATK
pipeline.transform(task_func=stages.base_recalibration_gatk,
name='base_recalibration_gatk',
input=output_from('local_realignment_gatk'),
filter=suffix('.sort.dedup.realn.bam'),
output=['.recal_data.csv', '.count_cov.log'])
# Print reads using GATK
(pipeline.transform(
task_func=stages.print_reads_gatk,
name='print_reads_gatk',
input=output_from('base_recalibration_gatk'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9_]+).recal_data.csv'),
add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.realn.bam'),
output='{path[0]}/{sample[0]}.sort.dedup.realn.recal.bam').follows(
'local_realignment_gatk'))
# Call variants using GATK
pipeline.transform(task_func=stages.call_variants_gatk,
name='call_variants_gatk',
input=output_from('print_reads_gatk'),
filter=suffix('.sort.dedup.realn.recal.bam'),
output='.raw.snps.indels.g.vcf')
# Combine G.VCF files for all samples using GATK
pipeline.merge(task_func=stages.combine_gvcf_gatk,
name='combine_gvcf_gatk',
input=output_from('call_variants_gatk'),
output='COMPLEXO.mergedgvcf.vcf')
# Genotype G.VCF files using GATK
pipeline.transform(task_func=stages.genotype_gvcf_gatk,
name='genotype_gvcf_gatk',
input=output_from('combine_gvcf_gatk'),
filter=suffix('.mergedgvcf.vcf'),
output='.genotyped.vcf')
# SNP recalibration using GATK
pipeline.transform(task_func=stages.snp_recalibrate_gatk,
name='snp_recalibrate_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.genotyped.vcf'),
output=['.snp_recal', '.snp_tranches', '.snp_plots.R'])
# INDEL recalibration using GATK
pipeline.transform(
task_func=stages.indel_recalibrate_gatk,
name='indel_recalibrate_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.genotyped.vcf'),
output=['.indel_recal', '.indel_tranches', '.indel_plots.R'])
# Apply SNP recalibration using GATK
(pipeline.transform(
task_func=stages.apply_snp_recalibrate_gatk,
name='apply_snp_recalibrate_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.genotyped.vcf'),
add_inputs=add_inputs(['COMPLEXO.snp_recal', 'COMPLEXO.snp_tranches']),
output='.recal_SNP.vcf').follows('snp_recalibrate_gatk'))
# Apply INDEL recalibration using GATK
(pipeline.transform(
task_func=stages.apply_indel_recalibrate_gatk,
name='apply_indel_recalibrate_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.genotyped.vcf'),
add_inputs=add_inputs(
['COMPLEXO.indel_recal', 'COMPLEXO.indel_tranches']),
output='.recal_INDEL.vcf').follows('indel_recalibrate_gatk'))
# Combine variants using GATK
(pipeline.transform(
task_func=stages.combine_variants_gatk,
name='combine_variants_gatk',
input=output_from('apply_snp_recalibrate_gatk'),
filter=suffix('.recal_SNP.vcf'),
add_inputs=add_inputs(['COMPLEXO.recal_INDEL.vcf']),
output='.combined.vcf').follows('apply_indel_recalibrate_gatk'))
# Select variants using GATK
pipeline.transform(task_func=stages.select_variants_gatk,
name='select_variants_gatk',
input=output_from('combine_variants_gatk'),
filter=suffix('.combined.vcf'),
output='.selected.vcf')
return pipeline
test_pipeline.transform(task3, task2, regex('(.*).2'), inputs([r"\1.2", tempdir + "a.1"]), r'\1.3')\
.posttask(lambda: do_write(test_file, "Task 3 Done\n"))
test_pipeline.transform(task4, tempdir + "*.1", suffix(".1"), ".4")\
.follows(task1)\
.posttask(lambda: do_write(test_file, "Task 4 Done\n"))\
.jobs_limit(1)
test_pipeline.files(task5, None, tempdir + 'a.5')\
.follows(mkdir(tempdir))\
.posttask(lambda: do_write(test_file, "Task 5 Done\n"))
test_pipeline.merge(task_func = task6,
input = [task3, task4, task5],
output = tempdir + "final.6")\
.follows(task3, task4, task5, ) \
.posttask(lambda: do_write(test_file, "Task 6 Done\n"))
def check_job_order_correct(filename):
"""
1 -> 2 -> 3 ->
-> 4 ->
5 -> 6
"""
precedence_rules = [[1, 2],
[2, 3],
[1, 4],
[5, 6],
def make_pipeline(state):
'''Build the pipeline by constructing stages and connecting them together'''
# Build an empty pipeline
pipeline = Pipeline(name='cellfree_seq')
# Stages are dependent on the state
stages = Stages(state)
safe_make_dir('alignments')
# The original FASTQ files
fastq_files = glob.glob('fastqs/*')
# This is a dummy stage. It is useful because it makes a node in the
# pipeline graph, and gives the pipeline an obvious starting point.
pipeline.originate(task_func=stages.original_fastqs,
name='original_fastqs',
output=fastq_files)
# Align paired end reads in FASTQ to the reference producing a BAM file
pipeline.transform(
task_func=stages.align_bwa,
name='align_bwa',
input=output_from('original_fastqs'),
# Match the R1 (read 1) FASTQ file and grab the path and sample name.
# This will be the first input to the stage.
filter=formatter('.+/(?P<sample>[a-zA-Z0-9_-]+)_R1.fastq.gz'),
# Add one more inputs to the stage:
# 1. The corresponding R2 FASTQ file
add_inputs=add_inputs('{path[0]}/{sample[0]}_R2.fastq.gz'),
# Add an "extra" argument to the state (beyond the inputs and outputs)
# which is the sample name. This is needed within the stage for finding out
# sample specific configuration options
extras=['{sample[0]}'],
# The output file name is the sample name with a .bam extension.
output='alignments/{sample[0]}.sort.hq.bam')
pipeline.transform(task_func=stages.run_connor,
name='run_connor',
input=output_from('align_bwa'),
filter=suffix('.sort.hq.bam'),
output='.sort.hq.connor.bam')
safe_make_dir('metrics')
safe_make_dir('metrics/summary')
safe_make_dir('metrics/connor')
pipeline.transform(
task_func=stages.intersect_bed,
name='intersect_bed_raw',
input=output_from('align_bwa'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9_-]+).sort.hq.bam'),
output='metrics/summary/{sample[0]}.intersectbed.bam')
pipeline.transform(task_func=stages.coverage_bed,
name='coverage_bed_raw',
input=output_from('intersect_bed_raw'),
filter=suffix('.intersectbed.bam'),
output='.bedtools_hist_all.txt')
pipeline.transform(
task_func=stages.genome_reads,
name='genome_reads_raw',
input=output_from('align_bwa'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9_-]+).sort.hq.bam'),
output='metrics/summary/{sample[0]}.mapped_to_genome.txt')
pipeline.transform(task_func=stages.target_reads,
name='target_reads_raw',
input=output_from('intersect_bed_raw'),
filter=suffix('.intersectbed.bam'),
output='.mapped_to_target.txt')
pipeline.transform(
task_func=stages.total_reads,
name='total_reads_raw',
input=output_from('align_bwa'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9_-]+).sort.hq.bam'),
output='metrics/summary/{sample[0]}.total_raw_reads.txt')
pipeline.collate(
task_func=stages.generate_stats,
name='generate_stats_raw',
input=output_from('coverage_bed_raw', 'genome_reads_raw',
'target_reads_raw', 'total_reads_raw'),
filter=regex(
r'.+/(.+)\.(bedtools_hist_all|mapped_to_genome|mapped_to_target|total_raw_reads)\.txt'
),
output=r'metrics/summary/all_sample.summary.\1.txt',
extras=[r'\1', 'summary.txt'])
pipeline.transform(
task_func=stages.intersect_bed,
name='intersect_bed_connor',
input=output_from('run_connor'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9_-]+).sort.hq.connor.bam'),
output='metrics/connor/{sample[0]}.intersectbed.bam')
pipeline.transform(task_func=stages.coverage_bed,
name='coverage_bed_connor',
input=output_from('intersect_bed_connor'),
filter=suffix('.intersectbed.bam'),
output='.bedtools_hist_all.txt')
pipeline.transform(
task_func=stages.genome_reads,
name='genome_reads_connor',
input=output_from('run_connor'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9_-]+).sort.hq.connor.bam'),
output='metrics/summary/{sample[0]}.mapped_to_genome.txt')
pipeline.transform(task_func=stages.target_reads,
name='target_reads_connor',
input=output_from('intersect_bed_connor'),
filter=suffix('.intersectbed.bam'),
output='.mapped_to_target.txt')
pipeline.transform(
task_func=stages.total_reads,
name='total_reads_connor',
input=output_from('run_connor'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9_-]+).sort.hq.connor.bam'),
output='metrics/summary/{sample[0]}.total_raw_reads.txt')
pipeline.collate(
task_func=stages.generate_stats,
name='generate_stats_connor',
input=output_from('coverage_bed_connor', 'genome_reads_connor',
'target_reads_connor', 'total_reads_connor'),
filter=regex(
r'.+/(.+)\.(bedtools_hist_all|mapped_to_genome|mapped_to_target|total_raw_reads)\.txt'
),
output=r'metrics/connor/all_sample.summary.\1.txt',
extras=[r'\1', 'connor.summary.txt'])
safe_make_dir('variants')
safe_make_dir('variants/vardict')
pipeline.transform(
task_func=stages.run_vardict,
name='run_vardict',
input=output_from('run_connor'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9_-]+).sort.hq.connor.bam'),
output='variants/vardict/{sample[0]}.vcf',
extras=['{sample[0]}'])
pipeline.transform(
task_func=stages.sort_vcfs,
name='sort_vcfs',
input=output_from('run_vardict'),
filter=formatter('variants/vardict/(?P<sample>[a-zA-Z0-9_-]+).vcf'),
output='variants/vardict/{sample[0]}.sorted.vcf.gz')
pipeline.transform(task_func=stages.index_vcfs,
name='index_vcfs',
input=output_from('sort_vcfs'),
filter=suffix('.sorted.vcf.gz'),
output='.sorted.vcf.gz.tbi')
(pipeline.merge(
task_func=stages.concatenate_vcfs,
name='concatenate_vcfs',
input=output_from('sort_vcfs'),
output='variants/vardict/combined.vcf.gz').follows('index_vcfs'))
pipeline.transform(task_func=stages.vt_decompose_normalise,
name='vt_decompose_normalise',
input=output_from('concatenate_vcfs'),
filter=suffix('.vcf.gz'),
output='.decomp.norm.vcf.gz')
pipeline.transform(task_func=stages.index_vcfs,
name='index_final_vcf',
input=output_from('vt_decompose_normalise'),
filter=suffix('.decomp.norm.vcf.gz'),
output='.decomp.norm.vcf.gz.tbi')
(pipeline.transform(
task_func=stages.apply_vep,
name='apply_vep',
input=output_from('vt_decompose_normalise'),
filter=suffix('.decomp.norm.vcf.gz'),
output='.decomp.norm.vep.vcf').follows('index_final_vcf'))
return pipeline
def make_pipeline(state):
'''Build the pipeline by constructing stages and connecting them together'''
# Build an empty pipeline
pipeline = Pipeline(name='hiplexpipe')
# Get a list of paths to all the FASTQ files
fastq_files = state.config.get_option('fastqs')
# Stages are dependent on the state
stages = Stages(state)
# The original FASTQ files
# This is a dummy stage. It is useful because it makes a node in the
# pipeline graph, and gives the pipeline an obvious starting point.
pipeline.originate(task_func=stages.original_fastqs,
name='original_fastqs',
output=fastq_files)
# Align paired end reads in FASTQ to the reference producing a BAM file
pipeline.transform(
task_func=stages.align_bwa,
name='align_bwa',
input=output_from('original_fastqs'),
# Match the R1 (read 1) FASTQ file and grab the path and sample name.
# This will be the first input to the stage.
# Hi-Plex example: OHI031002-P02F04_S318_L001_R1_001.fastq
# new sample name = OHI031002-P02F04
filter=formatter(
'.+/(?P<sample>[a-zA-Z0-9-]+)_(?P<readid>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_R1_(?P<lib>[a-zA-Z0-9-:]+).fastq'
),
# Add one more inputs to the stage:
# 1. The corresponding R2 FASTQ file
# Hi-Plex example: OHI031002-P02F04_S318_L001_R2_001.fastq
add_inputs=add_inputs(
'{path[0]}/{sample[0]}_{readid[0]}_{lane[0]}_R2_{lib[0]}.fastq'),
# Add an "extra" argument to the state (beyond the inputs and outputs)
# which is the sample name. This is needed within the stage for finding out
# sample specific configuration options
extras=['{sample[0]}', '{readid[0]}', '{lane[0]}', '{lib[0]}'],
# The output file name is the sample name with a .bam extension.
output='alignments/{sample[0]}_{readid[0]}/{sample[0]}_{readid[0]}.bam'
)
# Call variants using undr_rover
pipeline.transform(
task_func=stages.apply_undr_rover,
name='apply_undr_rover',
input=output_from('original_fastqs'),
# Match the R1 (read 1) FASTQ file and grab the path and sample name.
# This will be the first input to the stage.
filter=formatter(
'.+/(?P<sample>[a-zA-Z0-9-]+)_(?P<readid>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_R1_(?P<lib>[a-zA-Z0-9-:]+).fastq'
),
add_inputs=add_inputs(
'{path[0]}/{sample[0]}_{readid[0]}_{lane[0]}_R2_{lib[0]}.fastq'),
# extras=['{sample[0]}', '{readid[0]}', '{lane[0]}', '{lib[0]}'],
extras=['{sample[0]}', '{readid[0]}'],
# The output file name is the sample name with a .bam extension.
output='variants/undr_rover/{sample[0]}_{readid[0]}.vcf')
# Sort the BAM file using Picard
pipeline.transform(task_func=stages.sort_bam_picard,
name='sort_bam_picard',
input=output_from('align_bwa'),
filter=suffix('.bam'),
output='.sort.bam')
# High quality and primary alignments
pipeline.transform(task_func=stages.primary_bam,
name='primary_bam',
input=output_from('sort_bam_picard'),
filter=suffix('.sort.bam'),
output='.primary.bam')
# index bam file
pipeline.transform(task_func=stages.index_sort_bam_picard,
name='index_bam',
input=output_from('primary_bam'),
filter=suffix('.primary.bam'),
output='.primary.bam.bai')
# Clip the primer_seq from BAM File
(pipeline.transform(
task_func=stages.clip_bam,
name='clip_bam',
input=output_from('primary_bam'),
filter=suffix('.primary.bam'),
output='.primary.primerclipped.bam').follows('index_bam'))
###### GATK VARIANT CALLING ######
# Call variants using GATK
pipeline.transform(
task_func=stages.call_haplotypecaller_gatk,
name='call_haplotypecaller_gatk',
input=output_from('clip_bam'),
# filter=suffix('.merged.dedup.realn.bam'),
filter=formatter(
'.+/(?P<sample>[a-zA-Z0-9-_]+).primary.primerclipped.bam'),
output='variants/gatk/{sample[0]}.g.vcf')
# .follows('index_sort_bam_picard'))
# Combine G.VCF files for all samples using GATK
pipeline.merge(task_func=stages.combine_gvcf_gatk,
name='combine_gvcf_gatk',
input=output_from('call_haplotypecaller_gatk'),
output='variants/gatk/ALL.combined.vcf')
# Genotype G.VCF files using GATK
pipeline.transform(task_func=stages.genotype_gvcf_gatk,
name='genotype_gvcf_gatk',
input=output_from('combine_gvcf_gatk'),
filter=suffix('.combined.vcf'),
output='.raw.vcf')
# Annotate VCF file using GATK
pipeline.transform(task_func=stages.variant_annotator_gatk,
name='variant_annotator_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.raw.vcf'),
output='.raw.annotate.vcf')
# Apply VariantFiltration using GATK
pipeline.transform(task_func=stages.apply_variant_filtration_gatk,
name='apply_variant_filtration_gatk',
input=output_from('variant_annotator_gatk'),
filter=suffix('.raw.annotate.vcf'),
output='.raw.annotate.filtered.vcf')
# Apply NORM
(pipeline.transform(
task_func=stages.apply_vt,
name='apply_vt',
input=output_from('apply_variant_filtration_gatk'),
filter=suffix('.raw.annotate.filtered.vcf'),
# add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
output='.raw.annotate.filtered.norm.vcf').follows(
'apply_variant_filtration_gatk'))
# Apply VEP
(pipeline.transform(
task_func=stages.apply_vep,
name='apply_vep',
input=output_from('apply_vt'),
filter=suffix('.raw.annotate.filtered.norm.vcf'),
# add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
output='.raw.annotate.filtered.norm.vep.vcf').follows('apply_vt'))
# Apply SnpEff
(pipeline.transform(
task_func=stages.apply_snpeff,
name='apply_snpeff',
input=output_from('apply_vep'),
filter=suffix('.raw.annotate.filtered.norm.vep.vcf'),
# add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
output='.raw.annotate.filtered.norm.vep.snpeff.vcf').follows(
'apply_vep'))
# Apply vcfanno
(pipeline.transform(
task_func=stages.apply_vcfanno,
name='apply_vcfanno',
input=output_from('apply_snpeff'),
filter=suffix('.raw.annotate.filtered.norm.vep.snpeff.vcf'),
# add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
output='.annotated.vcf').follows('apply_snpeff'))
# Concatenate undr_rover vcf files
pipeline.merge(task_func=stages.apply_cat_vcf,
name='apply_cat_vcf',
input=output_from('apply_undr_rover'),
output='variants/undr_rover/ur.vcf.gz')
# # Apple VEP on concatenated undr_rover vcf file
# (pipeline.transform(
# task_func=stages.apply_vep,
# name='apply_vep_ur',
# input=output_from('apply_cat_vcf'),
# filter=suffix('.vcf.gz'),
# output='.vep.vcf')
# .follows('apply_cat_vcf'))
#
# # Apply vcfanno on concatenated/vep undr_rover vcf file
# (pipeline.transform(
# task_func=stages.apply_vcfanno,
# name='apply_vcfanno_ur',
# input=output_from('apply_vep_ur'),
# filter=suffix('.vep.vcf'),
# output='.vep.anno.vcf')
# .follows('apply_vep_ur'))
#
# # Apply snpeff
# (pipeline.transform(
# task_func=stages.apply_snpeff,
# name='apply_snpeff_ur',
# input=output_from('apply_vcfanno_ur'),
# filter=suffix('.vep.anno.vcf'),
# output='.vep.anno.snpeff.vcf.gz')
# .follows('apply_vcfanno_ur'))
#
# Apply tabix
pipeline.transform(task_func=stages.apply_tabix,
name='apply_tabix',
input=output_from('apply_cat_vcf'),
filter=suffix('.vcf.gz'),
output='.vcf.gz.tbi')
# # Apply HomopolymerRun
# (pipeline.transform(
# task_func=stages.apply_homopolymer_ann,
# name='apply_homopolymer_ann',
# input=output_from('apply_snpeff_ur'),
# filter=suffix('.vep.anno.snpeff.vcf.gz'),
# output='.annotated.vcf')
# .follows('apply_tabix'))
# # Apply summarize multi coverage
# (pipeline.merge(
# task_func=stages.apply_multicov,
# name='apply_multicov',
# input=output_from('primary_bam'),
# # filter=suffix('.primary.bam'),
# output='coverage/all.multicov.txt')
# .follows('index_bam'))
# Apply summarize picard coverage
# (pipeline.merge(
# task_func=stages.apply_summarize_picard,
# name='apply_summarize_picard',
# input=output_from('target_coverage'),
# output='coverage/all.hsmetrics.txt')
# .follows('target_coverage'))
# # Apply summarize multicov coverage plots
# (pipeline.merge(
# task_func=stages.apply_multicov_plots,
# name='apply_multicov_plots',
# input=output_from('apply_multicov'),
# output='coverage/coverage_analysis_main.html')
# .follows('apply_multicov'))
return pipeline
def make_pipeline(state):
'''Build the pipeline by constructing stages and connecting them together'''
# Build an empty pipeline
pipeline = Pipeline(name='complexo')
# Get a list of paths to all the FASTQ files
fastq_files = state.config.get_option('fastqs')
# Stages are dependent on the state
stages = Stages(state)
# The original FASTQ files
# This is a dummy stage. It is useful because it makes a node in the
# pipeline graph, and gives the pipeline an obvious starting point.
pipeline.originate(
task_func=stages.original_fastqs,
name='original_fastqs',
output=fastq_files)
# Align paired end reads in FASTQ to the reference producing a BAM file
pipeline.transform(
task_func=stages.align_bwa,
name='align_bwa',
input=output_from('original_fastqs'),
# Match the R1 (read 1) FASTQ file and grab the path and sample name.
# This will be the first input to the stage.
# We assume the sample name may consist of only alphanumeric
# characters.
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+)_R1.fastq.gz'),
# Add one more inputs to the stage:
# 1. The corresponding R2 FASTQ file
add_inputs=add_inputs('{path[0]}/{sample[0]}_R2.fastq.gz'),
# Add an "extra" argument to the state (beyond the inputs and outputs)
# which is the sample name. This is needed within the stage for finding out
# sample specific configuration options
extras=['{sample[0]}'],
# The output file name is the sample name with a .bam extension.
output='{path[0]}/{sample[0]}.bam')
# Sort the BAM file using Picard
pipeline.transform(
task_func=stages.sort_bam_picard,
name='sort_bam_picard',
input=output_from('align_bwa'),
filter=suffix('.bam'),
output='.sort.bam')
# Mark duplicates in the BAM file using Picard
pipeline.transform(
task_func=stages.mark_duplicates_picard,
name='mark_duplicates_picard',
input=output_from('sort_bam_picard'),
filter=suffix('.sort.bam'),
# XXX should make metricsup an extra output?
output=['.sort.dedup.bam', '.metricsdup'])
# Generate chromosome intervals using GATK
pipeline.transform(
task_func=stages.chrom_intervals_gatk,
name='chrom_intervals_gatk',
input=output_from('mark_duplicates_picard'),
filter=suffix('.sort.dedup.bam'),
output='.chr.intervals')
# Local realignment using GATK
(pipeline.transform(
task_func=stages.local_realignment_gatk,
name='local_realignment_gatk',
input=output_from('chrom_intervals_gatk'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).chr.intervals'),
add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.bam'),
output='{path[0]}/{sample[0]}.sort.dedup.realn.bam')
.follows('mark_duplicates_picard'))
# Base recalibration using GATK
pipeline.transform(
task_func=stages.base_recalibration_gatk,
name='base_recalibration_gatk',
input=output_from('local_realignment_gatk'),
filter=suffix('.sort.dedup.realn.bam'),
output=['.recal_data.csv', '.count_cov.log'])
# Print reads using GATK
(pipeline.transform(
task_func=stages.print_reads_gatk,
name='print_reads_gatk',
input=output_from('base_recalibration_gatk'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).recal_data.csv'),
add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.realn.bam'),
output='{path[0]}/{sample[0]}.sort.dedup.realn.recal.bam')
.follows('local_realignment_gatk'))
# Call variants using GATK
pipeline.transform(
task_func=stages.call_variants_gatk,
name='call_variants_gatk',
input=output_from('print_reads_gatk'),
filter=suffix('.sort.dedup.realn.recal.bam'),
output='.raw.snps.indels.g.vcf')
# Combine G.VCF files for all samples using GATK
pipeline.merge(
task_func=stages.combine_gvcf_gatk,
name='combine_gvcf_gatk',
input=output_from('call_variants_gatk'),
output='PCExomes.mergegvcf.vcf')
# Genotype G.VCF files using GATK
pipeline.transform(
task_func=stages.genotype_gvcf_gatk,
name='genotype_gvcf_gatk',
input=output_from('combine_gvcf_gatk'),
filter=suffix('.mergegvcf.vcf'),
output='.genotyped.vcf')
# SNP recalibration using GATK
pipeline.transform(
task_func=stages.snp_recalibrate_gatk,
name='snp_recalibrate_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.genotyped.vcf'),
output=['.snp_recal', '.snp_tranches', '.snp_plots.R'])
# INDEL recalibration using GATK
pipeline.transform(
task_func=stages.indel_recalibrate_gatk,
name='indel_recalibrate_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.genotyped.vcf'),
output=['.indel_recal', '.indel_tranches', '.indel_plots.R'])
# Apply SNP recalibration using GATK
(pipeline.transform(
task_func=stages.apply_snp_recalibrate_gatk,
name='apply_snp_recalibrate_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.genotyped.vcf'),
add_inputs=add_inputs(['PCExomes.snp_recal', 'PCExomes.snp_tranches']),
output='.recal_SNP.vcf')
.follows('snp_recalibrate_gatk'))
# Apply INDEL recalibration using GATK
(pipeline.transform(
task_func=stages.apply_indel_recalibrate_gatk,
name='apply_indel_recalibrate_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.genotyped.vcf'),
add_inputs=add_inputs(['PCExomes.indel_recal', 'PCExomes.indel_tranches']),
output='.recal_INDEL.vcf')
.follows('indel_recalibrate_gatk'))
# Combine variants using GATK
(pipeline.transform(
task_func=stages.combine_variants_gatk,
name='combine_variants_gatk',
input=output_from('apply_snp_recalibrate_gatk'),
filter=suffix('.recal_SNP.vcf'),
add_inputs=add_inputs(['PCExomes.recal_INDEL.vcf']),
output='.combined.vcf')
.follows('apply_indel_recalibrate_gatk'))
# Select variants using GATK
pipeline.transform(
task_func=stages.select_variants_gatk,
name='select_variants_gatk',
input=output_from('combine_variants_gatk'),
filter=suffix('.combined.vcf'),
output='.selected.vcf')
return pipeline
def make_pipeline(state):
"""Build the pipeline by constructing stages and connecting them together"""
# Build an empty pipeline
pipeline = Pipeline(name="crpipe")
# Get a list of paths to all the FASTQ files
fastq_files = state.config.get_option("fastqs")
# Find the path to the reference genome
# Stages are dependent on the state
stages = Stages(state)
# The original FASTQ files
# This is a dummy stage. It is useful because it makes a node in the
# pipeline graph, and gives the pipeline an obvious starting point.
pipeline.originate(task_func=stages.original_fastqs, name="original_fastqs", output=fastq_files)
# Convert FASTQ file to FASTA using fastx toolkit
# pipeline.transform(
# task_func=stages.fastq_to_fasta,
# name='fastq_to_fasta',
# input=output_from('original_fastqs'),
# filter=suffix('.fastq.gz'),
# output='.fasta')
# The original reference file
# This is a dummy stage. It is useful because it makes a node in the
# pipeline graph, and gives the pipeline an obvious starting point.
# pipeline.originate(
# task_func=stages.original_reference,
# name='original_reference',
# output=reference_file)
# Run fastQC on the FASTQ files
pipeline.transform(
task_func=stages.fastqc,
name="fastqc",
input=output_from("original_fastqs"),
filter=suffix(".fastq.gz"),
output="_fastqc",
)
# Index the reference using BWA
# pipeline.transform(
# task_func=stages.index_reference_bwa,
# name='index_reference_bwa',
# input=output_from('original_reference'),
# filter=suffix('.fa'),
# output=['.fa.amb', '.fa.ann', '.fa.pac', '.fa.sa', '.fa.bwt'])
# Index the reference using samtools
# pipeline.transform(
# task_func=stages.index_reference_samtools,
# name='index_reference_samtools',
# input=output_from('original_reference'),
# filter=suffix('.fa'),
# output='.fa.fai')
# Index the reference using bowtie 2
# pipeline.transform(
# task_func=stages.index_reference_bowtie2,
# name='index_reference_bowtie2',
# input=output_from('original_reference'),
# filter=formatter('.+/(?P<refname>[a-zA-Z0-9]+\.fa)'),
# output=['{path[0]}/{refname[0]}.1.bt2',
# '{path[0]}/{refname[0]}.2.bt2',
# '{path[0]}/{refname[0]}.3.bt2',
# '{path[0]}/{refname[0]}.4.bt2',
# '{path[0]}/{refname[0]}.rev.1.bt2',
# '{path[0]}/{refname[0]}.rev.2.bt2'],
# extras=['{path[0]}/{refname[0]}'])
# # Create a FASTA sequence dictionary for the reference using picard
# pipeline.transform(
# task_func=stages.reference_dictionary_picard,
# name='reference_dictionary_picard',
# input=output_from('original_reference'),
# filter=suffix('.fa'),
# output='.dict')
# Align paired end reads in FASTQ to the reference producing a BAM file
pipeline.transform(
task_func=stages.align_bwa,
name="align_bwa",
input=output_from("original_fastqs"),
# Match the R1 (read 1) FASTQ file and grab the path and sample name.
# This will be the first input to the stage.
# We assume the sample name may consist of only alphanumeric
# characters.
filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+)_R1.fastq.gz"),
# Add two more inputs to the stage:
# 1. The corresponding R2 FASTQ file
add_inputs=add_inputs("{path[0]}/{sample[0]}_R2.fastq.gz"),
# Add an "extra" argument to the state (beyond the inputs and outputs)
# which is the sample name. This is needed within the stage for finding out
# sample specific configuration options
extras=["{sample[0]}"],
# The output file name is the sample name with a .bam extension.
output="{path[0]}/{sample[0]}.bam",
)
# Sort alignment with sambamba
pipeline.transform(
task_func=stages.sort_bam_sambamba,
name="sort_alignment",
input=output_from("align_bwa"),
filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).bam"),
output="{path[0]}/{sample[0]}.sorted.bam",
)
# Extract MMR genes from the sorted BAM file
pipeline.transform(
task_func=stages.extract_genes_bedtools,
name="extract_genes_bedtools",
input=output_from("sort_alignment"),
filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).sorted.bam"),
output="{path[0]}/{sample[0]}.mmr.bam",
)
# Extract selected chromosomes from the sorted BAM file
pipeline.transform(
task_func=stages.extract_chromosomes_samtools,
name="extract_chromosomes_samtools",
input=output_from("sort_alignment"),
filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).sorted.bam"),
output="{path[0]}/{sample[0]}.chroms.bam",
)
# Index the MMR genes bam file with samtools
pipeline.transform(
task_func=stages.index_bam,
name="index_mmr_alignment",
input=output_from("extract_genes_bedtools"),
filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).mmr.bam"),
output="{path[0]}/{sample[0]}.mmr.bam.bai",
)
# Compute depth of coverage of the alignment with GATK DepthOfCoverage
# pipeline.transform(
# task_func=stages.alignment_coverage_gatk,
# name='alignment_coverage_gatk',
# input=output_from('sort_alignment'),
# filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'),
# add_inputs=add_inputs([reference_file]),
# output='{path[0]}/{sample[0]}.coverage_summary',
# extras=['{path[0]}/{sample[0]}_coverage'])
# Index the alignment with samtools
pipeline.transform(
task_func=stages.index_bam,
name="index_alignment",
input=output_from("sort_alignment"),
filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).sorted.bam"),
output="{path[0]}/{sample[0]}.sorted.bam.bai",
)
# Generate alignment stats with bamtools
pipeline.transform(
task_func=stages.bamtools_stats,
name="bamtools_stats",
input=output_from("align_bwa"),
filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).bam"),
output="{path[0]}/{sample[0]}.stats.txt",
)
# Extract the discordant paired-end alignments
pipeline.transform(
task_func=stages.extract_discordant_alignments,
name="extract_discordant_alignments",
input=output_from("align_bwa"),
filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).bam"),
output="{path[0]}/{sample[0]}.discordants.unsorted.bam",
)
# Extract split-read alignments
pipeline.transform(
task_func=stages.extract_split_read_alignments,
name="extract_split_read_alignments",
input=output_from("align_bwa"),
filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).bam"),
output="{path[0]}/{sample[0]}.splitters.unsorted.bam",
)
# Sort discordant reads.
# Samtools annoyingly takes the prefix of the output bam name as its argument.
# So we pass this as an extra argument. However Ruffus needs to know the full name
# of the output bam file, so we pass that as the normal output parameter.
pipeline.transform(
task_func=stages.sort_bam,
name="sort_discordants",
input=output_from("extract_discordant_alignments"),
filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).discordants.unsorted.bam"),
extras=["{path[0]}/{sample[0]}.discordants"],
output="{path[0]}/{sample[0]}.discordants.bam",
)
# Index the sorted discordant bam with samtools
# pipeline.transform(
# task_func=stages.index_bam,
# name='index_discordants',
# input=output_from('sort_discordants'),
# filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).discordants.bam'),
# output='{path[0]}/{sample[0]}.discordants.bam.bai')
# Sort discordant reads
# Samtools annoyingly takes the prefix of the output bam name as its argument.
# So we pass this as an extra argument. However Ruffus needs to know the full name
# of the output bam file, so we pass that as the normal output parameter.
pipeline.transform(
task_func=stages.sort_bam,
name="sort_splitters",
input=output_from("extract_split_read_alignments"),
filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).splitters.unsorted.bam"),
extras=["{path[0]}/{sample[0]}.splitters"],
output="{path[0]}/{sample[0]}.splitters.bam",
)
# Index the sorted splitters bam with samtools
# pipeline.transform(
# task_func=stages.index_bam,
# name='index_splitters',
# input=output_from('sort_splitters'),
# filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).splitters.bam'),
# output='{path[0]}/{sample[0]}.splitters.bam.bai')
# Call structural variants with lumpy
(
pipeline.transform(
task_func=stages.structural_variants_lumpy,
name="structural_variants_lumpy",
input=output_from("sort_alignment"),
filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).sorted.bam"),
add_inputs=add_inputs(["{path[0]}/{sample[0]}.splitters.bam", "{path[0]}/{sample[0]}.discordants.bam"]),
output="{path[0]}/{sample[0]}.lumpy.vcf",
)
.follows("index_alignment")
.follows("sort_splitters")
.follows("sort_discordants")
)
# Call genotypes on lumpy output using SVTyper
# (pipeline.transform(
# task_func=stages.genotype_svtyper,
# name='genotype_svtyper',
# input=output_from('structural_variants_lumpy'),
# filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).lumpy.vcf'),
# add_inputs=add_inputs(['{path[0]}/{sample[0]}.sorted.bam', '{path[0]}/{sample[0]}.splitters.bam']),
# output='{path[0]}/{sample[0]}.svtyper.vcf')
# .follows('align_bwa')
# .follows('sort_splitters')
# .follows('index_alignment')
# .follows('index_splitters')
# .follows('index_discordants'))
# Call SVs with Socrates
(
pipeline.transform(
task_func=stages.structural_variants_socrates,
name="structural_variants_socrates",
input=output_from("sort_alignment"),
filter=formatter(".+/(?P<sample>[a-zA-Z0-9]+).sorted.bam"),
# output goes to {path[0]}/socrates/
output="{path[0]}/socrates/results_Socrates_paired_{sample[0]}.sorted_long_sc_l25_q5_m5_i95.txt",
extras=["{path[0]}"],
)
)
# Call DELs with DELLY
pipeline.merge(
task_func=stages.deletions_delly,
name="deletions_delly",
input=output_from("sort_alignment"),
output="delly.DEL.vcf",
)
# Call DUPs with DELLY
pipeline.merge(
task_func=stages.duplications_delly,
name="duplications_delly",
input=output_from("sort_alignment"),
output="delly.DUP.vcf",
)
# Call INVs with DELLY
pipeline.merge(
task_func=stages.inversions_delly,
name="inversions_delly",
input=output_from("sort_alignment"),
output="delly.INV.vcf",
)
# Call TRAs with DELLY
pipeline.merge(
task_func=stages.translocations_delly,
name="translocations_delly",
input=output_from("sort_alignment"),
output="delly.TRA.vcf",
)
# Join both read pair files using gustaf_mate_joining
# pipeline.transform(
# task_func=stages.gustaf_mate_joining,
# name='gustaf_mate_joining',
# input=output_from('fastq_to_fasta'),
# # Match the R1 (read 1) FASTA file and grab the path and sample name.
# # This will be the first input to the stage.
# # We assume the sample name may consist of only alphanumeric
# # characters.
# filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+)_R1.fasta'),
# # Add one more input to the stage:
# # 1. The corresponding R2 FASTA file
# add_inputs=add_inputs(['{path[0]}/{sample[0]}_R2.fasta']),
# output='{path[0]}/{sample[0]}.joined_mates.fasta')
# Call structural variants with pindel
# (pipeline.transform(
# task_func=stages.structural_variants_pindel,
# name='structural_variants_pindel',
# input=output_from('sort_alignment'),
# filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'),
# add_inputs=add_inputs(['{path[0]}/{sample[0]}.pindel_config.txt', reference_file]),
# output='{path[0]}/{sample[0]}.pindel')
# .follows('index_reference_bwa')
# .follows('index_reference_samtools'))
return pipeline
def make_pipeline_process(state):
# Define empty pipeline
pipeline = Pipeline(name='hiplexpipe')
# Get a list of paths to all the directories to be combined for variant calling
run_directories = state.config.get_option('runs')
#grab files from each of the processed directories in "runs"
gatk_files = []
undr_rover_files = []
for directory in run_directories:
gatk_files.extend(glob.glob(directory + '/variants/gatk/*.g.vcf'))
undr_rover_files.extend(
glob.glob(directory + '/variants/undr_rover/*sorted.vcf.gz'))
# Stages are dependent on the state
stages = Stages(state)
# This is a dummy stage. It is useful because it makes a node in the
# pipeline graph, and gives the pipeline an obvious starting point.
pipeline.originate(task_func=stages.glob_gatk,
name='glob_gatk',
output=gatk_files)
#Dummy stage to grab the undr rover files
pipeline.originate(task_func=stages.glob_undr_rover,
name='glob_undr_rover',
output=undr_rover_files)
safe_make_dir('variants')
safe_make_dir('variants/gatk')
safe_make_dir('variants/undr_rover')
pipeline.merge(task_func=stages.concatenate_vcfs,
name='concatenate_vcfs',
input=output_from('glob_undr_rover'),
output='variants/undr_rover/combined_undr_rover.vcf.gz')
pipeline.transform(task_func=stages.index_final_vcf,
name='index_final_vcf',
input=output_from('concatenate_vcfs'),
filter=suffix('.vcf.gz'),
output='.vcf.gz.tbi')
# Combine G.VCF files for all samples using GATK
pipeline.merge(task_func=stages.combine_gvcf_gatk,
name='combine_gvcf_gatk',
input=output_from('glob_gatk'),
output='ALL.combined.vcf')
# Genotype G.VCF files using GATK
pipeline.transform(task_func=stages.genotype_gvcf_gatk,
name='genotype_gvcf_gatk',
input=output_from('combine_gvcf_gatk'),
filter=suffix('.combined.vcf'),
output='.raw.vcf')
# Apply GT filters to genotyped vcf
pipeline.transform(task_func=stages.genotype_filter_gatk,
name='genotype_filter_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.raw.vcf'),
output='.raw.gt-filter.vcf')
# Decompose and normalise multiallelic sites
pipeline.transform(task_func=stages.vt_decompose_normalise,
name='vt_decompose_normalise',
input=output_from('genotype_filter_gatk'),
filter=suffix('.raw.gt-filter.vcf'),
output='.raw.gt-filter.decomp.norm.vcf')
# Annotate VCF file using GATK
pipeline.transform(task_func=stages.variant_annotator_gatk,
name='variant_annotator_gatk',
input=output_from('vt_decompose_normalise'),
filter=suffix('.raw.gt-filter.decomp.norm.vcf'),
output='.raw.gt-filter.decomp.norm.annotate.vcf')
# Filter vcf
pipeline.transform(
task_func=stages.gatk_filter,
name='gatk_filter',
input=output_from('variant_annotator_gatk'),
filter=suffix('.raw.gt-filter.decomp.norm.annotate.vcf'),
output='.raw.gt-filter.decomp.norm.annotate.filter.vcf')
#Apply VEP
(pipeline.transform(
task_func=stages.apply_vep,
name='apply_vep',
input=output_from('gatk_filter'),
filter=suffix('.raw.gt-filter.decomp.norm.annotate.filter.vcf'),
add_inputs=add_inputs(
['variants/undr_rover/combined_undr_rover.vcf.gz']),
output='.raw.gt-filter.decomp.norm.annotate.filter.vep.vcf').follows(
'index_final_vcf'))
return pipeline
def make_pipeline(state):
'''Build the pipeline by constructing stages and connecting them together'''
# Build an empty pipeline
pipeline = Pipeline(name='thepipeline')
# Get a list of paths to all the FASTQ files
fastq_files = state.config.get_option('fastqs')
# Stages are dependent on the state
stages = Stages(state)
# The original FASTQ files
# This is a dummy stage. It is useful because it makes a node in the
# pipeline graph, and gives the pipeline an obvious starting point.
pipeline.originate(
task_func=stages.original_fastqs,
name='original_fastqs',
output=fastq_files)
# Align paired end reads in FASTQ to the reference producing a BAM file
pipeline.transform(
task_func=stages.align_bwa,
name='align_bwa',
input=output_from('original_fastqs'),
# Match the R1 (read 1) FASTQ file and grab the path and sample name.
# This will be the first input to the stage.
# We assume the sample name may consist of only alphanumeric
# characters.
# filter=formatter('(?P<path>.+)/(?P<readid>[a-zA-Z0-9-\.]+)_(?P<lib>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9]+)_1.fastq.gz'),
# 1_HFYLVCCXX:2:TCCGCGAA_2_GE0343_1.fastq.gz
# 1_HCJWFBCXX:GGACTCCT_L001_9071584415739518822-AGRF-023_R2.fastq.gz
filter=formatter(
'.+/(?P<readid>[a-zA-Z0-9-]+)_(?P<lib>[a-zA-Z0-9-:]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9-]+)_R1.fastq.gz'),
# Add one more inputs to the stage:
# 1. The corresponding R2 FASTQ file
# e.g. C2WPF.5_Solexa-201237_5_X4311_1.fastq.gz
add_inputs=add_inputs(
'{path[0]}/{readid[0]}_{lib[0]}_{lane[0]}_{sample[0]}_R2.fastq.gz'),
# Add an "extra" argument to the state (beyond the inputs and outputs)
# which is the sample name. This is needed within the stage for finding out
# sample specific configuration options
extras=['{readid[0]}', '{lib[0]}', '{lane[0]}', '{sample[0]}'],
# extras=['{sample[0]}'],
# The output file name is the sample name with a .bam extension.
output='alignments/{sample[0]}/{readid[0]}_{lib[0]}_{lane[0]}_{sample[0]}.bam')
# Sort the BAM file using Picard
pipeline.transform(
task_func=stages.sort_bam_picard,
name='sort_bam_picard',
input=output_from('align_bwa'),
filter=suffix('.bam'),
output='.sort.bam')
# Mark duplicates in the BAM file using Picard
pipeline.transform(
task_func=stages.mark_duplicates_picard,
name='mark_duplicates_picard',
input=output_from('sort_bam_picard'),
filter=suffix('.sort.bam'),
# XXX should make metricsup an extra output?
output=['.sort.dedup.bam', '.metricsdup'])
# Local realignment using GATK
# Generate RealignerTargetCreator using GATK
pipeline.transform(
task_func=stages.realigner_target_creator,
name='realigner_target_creator',
input=output_from('mark_duplicates_picard'),
filter=suffix('.sort.dedup.bam'),
output='.intervals')
# Local realignment using GATK
(pipeline.transform(
task_func=stages.local_realignment_gatk,
name='local_realignment_gatk',
input=output_from('realigner_target_creator'),
# filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).chr.intervals'),
filter=formatter(
'.+/(?P<readid>[a-zA-Z0-9-]+)_(?P<lib>[a-zA-Z0-9-:]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9-]+).intervals'),
# add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.bam'),
add_inputs=add_inputs(
'alignments/{sample[0]}/{readid[0]}_{lib[0]}_{lane[0]}_{sample[0]}.sort.dedup.bam'),
output='alignments/{sample[0]}/{readid[0]}_{lib[0]}_{lane[0]}_{sample[0]}.sort.dedup.realn.bam')
.follows('mark_duplicates_picard'))
# Base recalibration using GATK
pipeline.transform(
task_func=stages.base_recalibration_gatk,
name='base_recalibration_gatk',
input=output_from('local_realignment_gatk'),
filter=suffix('.sort.dedup.realn.bam'),
output=['.recal_data.csv', '.count_cov.log'])
# Print reads using GATK
(pipeline.transform(
task_func=stages.print_reads_gatk,
name='print_reads_gatk',
input=output_from('base_recalibration_gatk'),
# filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).recal_data.csv'),
filter=formatter(
# '.+/(?P<readid>[a-zA-Z0-9-\.]+)_(?P<lib>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9]+).recal_data.csv'),
'.+/(?P<readid>[a-zA-Z0-9-]+)_(?P<lib>[a-zA-Z0-9-:]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9-]+).recal_data.csv'),
# '.+/(?P<readid>[a-zA-Z0-9-]+)_(?P<lib>[a-zA-Z0-9-:]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9-]+).recal_data.csv'),
# add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.realn.bam'),
add_inputs=add_inputs(
'alignments/{sample[0]}/{readid[0]}_{lib[0]}_{lane[0]}_{sample[0]}.sort.dedup.realn.bam'),
# output='{path[0]}/{sample[0]}.sort.dedup.realn.recal.bam')
output='alignments/{sample[0]}/{readid[0]}_{lib[0]}_{lane[0]}_{sample[0]}.sort.dedup.realn.recal.bam')
.follows('local_realignment_gatk'))
# Merge lane bams to sample bams
pipeline.collate(
task_func=stages.merge_sample_bams,
name='merge_sample_bams',
filter=formatter(
# '.+/(?P<readid>[a-zA-Z0-9-\.]+)_(?P<lib>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9]+).sort.dedup.realn.recal.bam'),
'.+/(?P<readid>[a-zA-Z0-9-]+)_(?P<lib>[a-zA-Z0-9-:]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9-]+).sort.dedup.realn.recal.bam'),
# inputs=add_inputs('alignments/{sample[0]}/{readid[0]}_{lib[0]}_{lane[0]}_{sample[0]}.sort.dedup.realn.bam'),
input=output_from('print_reads_gatk'),
output='alignments/{sample[0]}/{sample[0]}.merged.bam')
# Mark duplicates in the BAM file using Picard
pipeline.transform(
task_func=stages.mark_duplicates_picard,
name='mark_duplicates_picard2',
input=output_from('merge_sample_bams'),
# filter=formatter(
# '.+/(?P<readid>[a-zA-Z0-9-\.]+)_(?P<lib>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9]+).merged.bam'),
filter=suffix('.merged.bam'),
# XXX should make metricsup an extra output?
output=['.merged.dedup.bam', '.metricsdup'])
# Local realignment2 using GATK
# Generate RealignerTargetCreator using GATK
pipeline.transform(
task_func=stages.realigner_target_creator,
name='realigner_target_creator2',
input=output_from('mark_duplicates_picard2'),
filter=suffix('.dedup.bam'),
output='.intervals')
# Local realignment using GATK
(pipeline.transform(
task_func=stages.local_realignment_gatk,
name='local_realignment_gatk2',
input=output_from('realigner_target_creator2'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9-]+).merged.intervals'),
# filter=formatter(
# '.+/(?P<readid>[a-zA-Z0-9-\.]+)_(?P<lib>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_(?P<sample>[a-zA-Z0-9]+).intervals'),
# add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.bam'),
add_inputs=add_inputs(
'alignments/{sample[0]}/{sample[0]}.merged.dedup.bam'),
output='alignments/{sample[0]}/{sample[0]}.merged.dedup.realn.bam')
.follows('mark_duplicates_picard2'))
# Call variants using GATK
pipeline.transform(
task_func=stages.call_haplotypecaller_gatk,
name='call_haplotypecaller_gatk',
input=output_from('local_realignment_gatk2'),
# filter=suffix('.merged.dedup.realn.bam'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9-]+).merged.dedup.realn.bam'),
output='variants/{sample[0]}.g.vcf')
# Combine G.VCF files for all samples using GATK
pipeline.merge(
task_func=stages.combine_gvcf_gatk,
name='combine_gvcf_gatk',
input=output_from('call_haplotypecaller_gatk'),
output='variants/ALL.combined.vcf')
# Genotype G.VCF files using GATK
pipeline.transform(
task_func=stages.genotype_gvcf_gatk,
name='genotype_gvcf_gatk',
input=output_from('combine_gvcf_gatk'),
filter=suffix('.combined.vcf'),
output='.raw.vcf')
# SNP recalibration using GATK
pipeline.transform(
task_func=stages.snp_recalibrate_gatk,
name='snp_recalibrate_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.raw.vcf'),
output=['.snp_recal', '.snp_tranches', '.snp_plots.R'])
# INDEL recalibration using GATK
pipeline.transform(
task_func=stages.indel_recalibrate_gatk,
name='indel_recalibrate_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.raw.vcf'),
output=['.indel_recal', '.indel_tranches', '.indel_plots.R'])
# Apply SNP recalibration using GATK
(pipeline.transform(
task_func=stages.apply_snp_recalibrate_gatk,
name='apply_snp_recalibrate_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.raw.vcf'),
add_inputs=add_inputs(['ALL.snp_recal', 'ALL.snp_tranches']),
output='.recal_SNP.vcf')
.follows('snp_recalibrate_gatk'))
# Apply INDEL recalibration using GATK
(pipeline.transform(
task_func=stages.apply_indel_recalibrate_gatk,
name='apply_indel_recalibrate_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.raw.vcf'),
add_inputs=add_inputs(
['ALL.indel_recal', 'ALL.indel_tranches']),
output='.recal_INDEL.vcf')
.follows('indel_recalibrate_gatk'))
# Combine variants using GATK
(pipeline.transform(
task_func=stages.combine_variants_gatk,
name='combine_variants_gatk',
input=output_from('apply_snp_recalibrate_gatk'),
filter=suffix('.recal_SNP.vcf'),
add_inputs=add_inputs(['ALL.recal_INDEL.vcf']),
# output='.combined.vcf')
output='ALL.raw.vqsr.vcf')
.follows('apply_indel_recalibrate_gatk'))
#
# # Select variants using GATK
# pipeline.transform(
# task_func=stages.select_variants_gatk,
# name='select_variants_gatk',
# input=output_from('combine_variants_gatk'),
# filter=suffix('.combined.vcf'),
# output='.selected.vcf')
return pipeline
output=[tempdir + "/g_name.tmp1", tempdir + "/h_name.tmp1"])
test_pipeline1.product(task_func=check_product_task,
input=[tempdir + "/" + prefix +
"_name.tmp1" for prefix in "abcd"],
filter=formatter(".*/(?P<FILE_PART>.+).tmp1$"),
input2=generate_initial_files2,
filter2=formatter(),
input3=generate_initial_files3,
filter3=formatter(r"tmp1$"),
output="{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.{basename[2][0]}.tmp2",
extras=["{basename[0][0][0]}{basename[1][0][0]}{basename[2][0][0]}", # extra: prefices only (abcd etc)
# extra: path for 2nd input, 1st file
"{subpath[0][0][0]}",
"{subdir[0][0][0]}"]).follows("WOWWWEEE").follows(gen_task1).follows(generate_initial_files1).follows("generate_initial_files1")
test_pipeline1.merge(task_func=check_product_merged_task,
input=check_product_task,
output=tempdir + "/merged.results")
test_pipeline1.product(task_func=check_product_misspelt_capture_error_task,
input=gen_task1,
filter=formatter(".*/(?P<FILE_PART>.+).tmp1$"),
output="{path[0][0]}/{FILEPART[0][0]}.tmp2")
test_pipeline1.product(task_func=check_product_out_of_range_formatter_ref_error_task,
input=generate_initial_files1, #
filter=formatter(".*/(?P<FILE_PART>.+).tmp1$"),
output="{path[2][0]}/{basename[0][0]}.tmp2",
extras=["{FILE_PART[0][0]}"])
test_pipeline1.product(task_func=check_product_formatter_ref_index_error_task,
input=output_from("generate_initial_files1"),
filter=formatter(".*/(?P<FILE_PART>.+).tmp1$"),
output="{path[0][0][1000]}/{basename[0][0]}.tmp2",
extras=["{FILE_PART[0][0]}"])
def make_pipeline(state):
'''Build the pipeline by constructing stages and connecting them together'''
# Build an empty pipeline
pipeline = Pipeline(name='hiplexpipe')
# Get a list of paths to all the FASTQ files
fastq_files = state.config.get_option('fastqs')
# Stages are dependent on the state
stages = Stages(state)
# The original FASTQ files
# This is a dummy stage. It is useful because it makes a node in the
# pipeline graph, and gives the pipeline an obvious starting point.
pipeline.originate(task_func=stages.original_fastqs,
name='original_fastqs',
output=fastq_files)
# Align paired end reads in FASTQ to the reference producing a BAM file
pipeline.transform(
task_func=stages.align_bwa,
name='align_bwa',
input=output_from('original_fastqs'),
# Match the R1 (read 1) FASTQ file and grab the path and sample name.
# This will be the first input to the stage.
# Hi-Plex example: OHI031002-P02F04_S318_L001_R1_001.fastq
# new sample name = OHI031002-P02F04
filter=formatter(
'.+/(?P<sample>[a-zA-Z0-9-]+)_(?P<readid>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_R1_(?P<lib>[a-zA-Z0-9-:]+).fastq'
),
# Add one more inputs to the stage:
# 1. The corresponding R2 FASTQ file
# Hi-Plex example: OHI031002-P02F04_S318_L001_R2_001.fastq
add_inputs=add_inputs(
'{path[0]}/{sample[0]}_{readid[0]}_{lane[0]}_R2_{lib[0]}.fastq'),
# Add an "extra" argument to the state (beyond the inputs and outputs)
# which is the sample name. This is needed within the stage for finding out
# sample specific configuration options
extras=['{sample[0]}', '{readid[0]}', '{lane[0]}', '{lib[0]}'],
# The output file name is the sample name with a .bam extension.
output='alignments/{sample[0]}_{readid[0]}/{sample[0]}_{readid[0]}.bam'
)
# Call variants using undr_rover
pipeline.transform(
task_func=stages.apply_undr_rover,
name='apply_undr_rover',
input=output_from('original_fastqs'),
# Match the R1 (read 1) FASTQ file and grab the path and sample name.
# This will be the first input to the stage.
filter=formatter(
'.+/(?P<sample>[a-zA-Z0-9-]+)_(?P<readid>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_R1_(?P<lib>[a-zA-Z0-9-:]+).fastq'
),
add_inputs=add_inputs(
'{path[0]}/{sample[0]}_{readid[0]}_{lane[0]}_R2_{lib[0]}.fastq'),
# extras=['{sample[0]}', '{readid[0]}', '{lane[0]}', '{lib[0]}'],
extras=['{sample[0]}', '{readid[0]}'],
# The output file name is the sample name with a .bam extension.
output='variants/undr_rover/{sample[0]}_{readid[0]}.vcf')
# Sort the BAM file using Picard
pipeline.transform(task_func=stages.sort_bam_picard,
name='sort_bam_picard',
input=output_from('align_bwa'),
filter=suffix('.bam'),
output='.sort.bam')
# High quality and primary alignments
pipeline.transform(task_func=stages.primary_bam,
name='primary_bam',
input=output_from('sort_bam_picard'),
filter=suffix('.sort.bam'),
output='.primary.bam')
# index bam file
pipeline.transform(task_func=stages.index_sort_bam_picard,
name='index_bam',
input=output_from('primary_bam'),
filter=suffix('.primary.bam'),
output='.primary.bam.bai')
# Clip the primer_seq from BAM File
(pipeline.transform(
task_func=stages.clip_bam,
name='clip_bam',
input=output_from('primary_bam'),
filter=suffix('.primary.bam'),
output='.primary.primerclipped.bam').follows('index_bam'))
###### GATK VARIANT CALLING ######
# Call variants using GATK
pipeline.transform(
task_func=stages.call_haplotypecaller_gatk,
name='call_haplotypecaller_gatk',
input=output_from('clip_bam'),
# filter=suffix('.merged.dedup.realn.bam'),
filter=formatter(
'.+/(?P<sample>[a-zA-Z0-9-_]+).primary.primerclipped.bam'),
output='variants/gatk/{sample[0]}.g.vcf')
# .follows('index_sort_bam_picard'))
# Combine G.VCF files for all samples using GATK
pipeline.merge(task_func=stages.combine_gvcf_gatk,
name='combine_gvcf_gatk',
input=output_from('call_haplotypecaller_gatk'),
output='variants/gatk/ALL.combined.vcf')
# Genotype G.VCF files using GATK
pipeline.transform(task_func=stages.genotype_gvcf_gatk,
name='genotype_gvcf_gatk',
input=output_from('combine_gvcf_gatk'),
filter=suffix('.combined.vcf'),
output='.raw.vcf')
# Annotate VCF file using GATK
pipeline.transform(task_func=stages.variant_annotator_gatk,
name='variant_annotator_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.raw.vcf'),
output='.raw.annotate.vcf')
# Apply VariantFiltration using GATK
pipeline.transform(task_func=stages.apply_variant_filtration_gatk_lenient,
name='apply_variant_filtration_gatk_lenient',
input=output_from('variant_annotator_gatk'),
filter=suffix('.raw.annotate.vcf'),
output='.raw.annotate.filtered_lenient.vcf')
return pipeline
input=[tempdir + "/" + prefix + "_name.tmp1" for prefix in "abcd"],
filter=formatter(".*/(?P<FILE_PART>.+).tmp1$"),
input2=generate_initial_files2,
filter2=formatter(),
input3=generate_initial_files3,
filter3=formatter(r"tmp1$"),
output=
"{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.{basename[2][0]}.tmp2",
extras=[
"{basename[0][0][0]}{basename[1][0][0]}{basename[2][0][0]}", # extra: prefices only (abcd etc)
"{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
"{subdir[0][0][0]}"
]).follows("WOWWWEEE").follows(gen_task1).follows(
generate_initial_files1).follows("generate_initial_files1")
test_pipeline1.merge(task_func=test_product_merged_task,
input=test_product_task,
output=tempdir + "/merged.results")
test_pipeline1.product(task_func=test_product_misspelt_capture_error_task,
input=gen_task1,
filter=formatter(".*/(?P<FILE_PART>.+).tmp1$"),
output="{path[0][0]}/{FILEPART[0][0]}.tmp2")
test_pipeline1.product(
task_func=test_product_out_of_range_formatter_ref_error_task,
input=generate_initial_files1, #
filter=formatter(".*/(?P<FILE_PART>.+).tmp1$"),
output="{path[2][0]}/{basename[0][0]}.tmp2",
extras=["{FILE_PART[0][0]}"])
test_pipeline1.product(task_func=test_product_formatter_ref_index_error_task,
input=output_from("generate_initial_files1"),
filter=formatter(".*/(?P<FILE_PART>.+).tmp1$"),
output="{path[0][0][1000]}/{basename[0][0]}.tmp2",
test_pipeline.transform(task3, task2, regex('(.*).2'), inputs([r"\1.2", tempdir + "a.1"]), r'\1.3')\
.posttask(lambda: do_write(test_file, "Task 3 Done\n"))
test_pipeline.transform(task4, tempdir + "*.1", suffix(".1"), ".4")\
.follows(task1)\
.posttask(lambda: do_write(test_file, "Task 4 Done\n"))\
.jobs_limit(1)
test_pipeline.files(task5, None, tempdir + 'a.5')\
.follows(mkdir(tempdir))\
.posttask(lambda: do_write(test_file, "Task 5 Done\n"))
test_pipeline.merge(task_func = task6,
input = [task3, task4, task5],
output = tempdir + "final.6")\
.follows(task3, task4, task5, ) \
.posttask(lambda: do_write(test_file, "Task 6 Done\n"))
def check_job_order_correct(filename):
"""
1 -> 2 -> 3 ->
-> 4 ->
5 -> 6
"""
precedence_rules = [[1, 2], [2, 3], [1, 4], [5, 6], [3, 6], [4, 6]]
index_re = re.compile(r'.*\.([0-9])["\]\n]*$')
job_indices = defaultdict(list)
def make_pipeline(state):
'''Build the pipeline by constructing stages and connecting them together'''
# Build an empty pipeline
pipeline = Pipeline(name="radpipe")
# Stages are dependent on the state
stages = PipelineStages(state)
# Get a list of library objects.
libraries = parse_libraries(
libraries=state.config.get_options("libraries"))
# Get a list of input files
input_files = [l.files for l in libraries]
# input_files = [item for sublist in input_files for item in sublist]
state.logger.info("Input files: " + str(input_files))
# Get a list of all samples for each library
samples_dict = OrderedDict()
for l in libraries:
samples_dict[l.name] = l.samples
state.logger.debug("Samples: " + str(samples_dict))
# Make sure that there are no duplicate samples
sample_list = [
item for sublist in samples_dict.values() for item in sublist
]
sample_counts = Counter(sample_list)
for sample in sample_counts:
if sample_counts[sample] > 1:
print("Sample {} appears {} times in the barcodes files. "
"Sample names must be unique".format(sample,
sample_counts[sample]))
sys.exit(radpipe.error_codes.INVALID_INPUT_FILE)
# Define output directories
output_dir = get_output_paths(state)
state.logger.debug(output_dir)
# Allow multiple comma-separated tasks
if len(state.options.target_tasks) == 1:
state.options.target_tasks = state.options.target_tasks[0].split(",")
if len(state.options.forced_tasks) == 1:
state.options.forced_tasks = state.options.forced_tasks[0].split(",")
state.logger.debug("Target tasks: " + str(state.options.target_tasks))
state.logger.debug("Forced tasks: " + str(state.options.forced_tasks))
# Check if alignment_method is valid
alignment_method = state.config.get_options(
"alignment_method").strip().lower()
if alignment_method not in ["bwa mem", "bowtie"]:
print("Error: Invalid alignment_method in config file. " \
"Valid options are ['bwa mem', 'bowtie'].")
sys.exit(radpipe.error_codes.INVALID_ARGUMENT)
if alignment_method == "bwa mem":
align_task_name = "bwa_mem"
index_task_name = "bwa_index"
else:
align_task_name = "bowtie"
index_task_name = "bowtie_index"
# TODO: Refactor this
# If 'alignment' is in target_tasks or forced_tasks, specify which
# type of alignment job
if "alignment" in state.options.target_tasks:
index = state.options.target_tasks.index("alignment")
state.options.target_tasks[index] = align_task_name
if "alignment" in state.options.forced_tasks:
index = state.options.forced_tasks.index("alignment")
state.options.forced_tasks[index] = align_task_name
# If 'build_index' is in target_tasks or forced_tasks, specify which
# type of index job
if "build_index" in state.options.target_tasks:
index = state.options.target_tasks.index("build_index")
state.options.target_tasks[index] = index_task_name
if "build_index" in state.options.forced_tasks:
index = state.options.forced_tasks.index("build_index")
state.options.forced_tasks[index] = index_task_name
state.logger.debug(state)
# Whether to include filter_bam stage or not
filter_bams = False
try:
samtools_view_options = state.config.get_options(
"samtools_view_options")
if samtools_view_options:
filter_bams = True
except:
pass
state.logger.info("Filter bams: {}".format(filter_bams))
# Population map filenames
popmap_file = "{output_dir}/{name}_popmap.txt".format(
output_dir=output_dir["populations"],
name=state.config.get_options("analysis_id"))
try:
config_popmap_file = state.config.get_options("popmap_file")
if config_popmap_file:
state.logger.info(
"Using popmap file: {}".format(config_popmap_file))
else:
raise (Exception)
except Exception:
config_popmap_file = None
state.logger.info("Creating new popmap file: {}".format(popmap_file))
# Population r values
populations_r = state.config.get_options("populations_r")
assert (isinstance(populations_r, list))
# Dummy stages. These do nothing except provide a node at the beginning
# for the pipeline graph, giving the pipeline an obvious starting point.
pipeline.originate(task_func=stages.do_nothing,
name="original_fastqs",
output=input_files)
pipeline.originate(task_func=stages.do_nothing,
name="reference_genome",
output=state.config.get_options("reference_genome"))
# Create a copy of the population map file needed for stacks, or create
# one denovo using the sample list.
pipeline.originate(task_func=stages.create_popmap_file,
name="create_popmap_file",
output=[popmap_file],
extras=[config_popmap_file, sample_list])
# Create index for reference genome based on alignment method.
if alignment_method == "bwa mem":
pipeline.transform(
task_func=stages.bwa_index,
name="bwa_index",
input=output_from("reference_genome"),
filter=formatter(".+/(?P<ref>[^/]+).(fa|fasta)"),
output=path_list_join(output_dir["reference"],
["reference.fa.bwt", "reference.fa.sa"]),
extras=[output_dir["reference"]])
if alignment_method == "bowtie":
pipeline.transform(task_func=stages.bowtie_index,
name="bowtie_index",
input=output_from("reference_genome"),
filter=formatter(".+/(?P<ref>[^/]+).(fa|fasta)"),
output=path_list_join(
output_dir["reference"],
["reference.1.ebwt", "reference.rev.1.ebwt"]),
extras=[output_dir["reference"]])
# FastQC
pipeline.transform(
task_func=stages.fastqc,
name="fastqc",
input=output_from("original_fastqs"),
filter=formatter(".+/(?P<lib>[^/]+)/(?P<fn>[^/]+).(fastq|fq).gz"),
output="%s/{lib[0]}/{fn[0]}_fastqc.zip" % output_dir["fastqc"],
extras=[output_dir["fastqc"], "{lib[0]}"])
# MultiQC: FastQC
pipeline.merge(task_func=stages.multiqc_fastqc,
name="multiqc_fastqc",
input=output_from("fastqc"),
output="%s/multiqc_fastqc_report.html" % output_dir["qc"],
extras=[output_dir["qc"], output_dir["fastqc"]])
# Stacks: Process RAD-Tags
pipeline.transform(task_func=stages.process_radtags,
name="process_radtags",
input=output_from("original_fastqs"),
filter=formatter(".+/(?P<lib>[^/]+)/[^/]+"),
output="%s/{lib[0]}/{lib[0]}.success" %
output_dir["process_radtags"],
extras=[
output_dir["process_radtags"], "{lib[0]}",
state.config.get_options("renz_1"),
state.config.get_options("renz_2"),
state.config.get_options("process_radtags_options")
])
# Create a list for alignment with the input fastq files from process_radtags
process_radtags_outputs = []
for l in libraries:
for s in l.samples:
base = "{dir}/{lib}/{sample}".format(
dir=output_dir["process_radtags"], lib=l.lib_id, sample=s)
process_radtags_outputs.append(
[base + ".1.fq.gz", base + ".2.fq.gz"])
# print(process_radtags_outputs)
# Alignment
if align_task_name == "bwa_mem":
(pipeline.transform(
task_func=stages.bwa_align,
name=align_task_name,
input=process_radtags_outputs,
filter=formatter(".+/(?P<sm>[^/]+).1.fq.gz"),
output="%s/{sm[0]}.bwa.bam" % output_dir["alignments"],
extras=[
os.path.join(output_dir["reference"], "reference.fa"),
"{path[0]}", output_dir["alignments"], "{sm[0]}",
state.config.get_options("alignment_options")
])).follows("bwa_index").follows("process_radtags")
if align_task_name == "bowtie":
(pipeline.transform(
task_func=stages.bowtie_align,
name=align_task_name,
input=process_radtags_outputs,
filter=formatter(".+/(?P<sm>[^/]+).1.fq.gz"),
output="%s/{sm[0]}.bowtie.bam" % output_dir["alignments"],
extras=[
os.path.join(output_dir["reference"], "reference"),
"{path[0]}", output_dir["alignments"], "{sm[0]}",
state.config.get_options("alignment_options")
])).follows("bowtie_index").follows("process_radtags")
# Sort BAM and index
pipeline.transform(task_func=stages.sort_bam,
name="sort_bam",
input=output_from(align_task_name),
filter=suffix(".bam"),
output=".sorted.bam")
if filter_bams:
final_bam_task_name = "filter_bam"
pipeline.transform(
task_func=stages.filter_bam,
name="filter_bam",
input=output_from("sort_bam"),
filter=suffix(".sorted.bam"),
output=".sorted.filtered.bam",
extras=[state.config.get_options("samtools_view_options")])
else:
final_bam_task_name = "sort_bam"
# Samtools flagstat
pipeline.transform(task_func=stages.flagstat,
name="flagstat",
input=output_from(final_bam_task_name),
filter=suffix(".bam"),
output=".flagstat.txt",
output_dir=output_dir["flagstat"])
# MultiQC: flagstat
pipeline.merge(task_func=stages.multiqc_flagstat,
name="multiqc_flagstat",
input=output_from("flagstat"),
output="%s/multiqc_flagstat_report.html" % output_dir["qc"],
extras=[output_dir["qc"], output_dir["flagstat"]])
# Stacks: gstacks
pipeline.merge(task_func=stages.gstacks,
name="gstacks",
input=output_from(final_bam_task_name),
output="%s/catalog.fa.gz" % output_dir["gstacks"],
extras=[
output_dir["alignments"], output_dir["gstacks"],
align_task_name, final_bam_task_name, sample_list,
state.config.get_options("gstacks_options")
])
# Define outputs from each run of populations
populations_outputs = []
for r in populations_r:
dir_name = "{pop_dir}/{analysis_name}_r{r}".format(
pop_dir=output_dir["populations"],
analysis_name=state.config.get_options("analysis_id"),
r=r)
populations_outputs.append(
os.path.join(dir_name, "populations.snps.vcf"))
# print(populations_outputs)
# Stacks: populations
pipeline.originate(task_func=stages.populations,
name="popluations",
output=populations_outputs,
extras=[
output_dir["gstacks"], output_dir["populations"],
popmap_file,
state.config.get_options("populations_options")
]).follows("gstacks").follows("create_popmap_file")
return pipeline
null -> "test_active_if/b.1"
"test_active_if/b.1" -> "test_active_if/b.2"
"test_active_if/b.2" -> "test_active_if/b.4"
"test_active_if/b.4" -> "test_active_if/summary.5"
"""
# alternative syntax
test_pipeline = Pipeline("test")
test_pipeline.originate(task1, ['test_active_if/a.1', 'test_active_if/b.1'], "an extra_parameter")\
.follows(mkdir("test_active_if"))
test_pipeline.transform(task2, task1, suffix(".1"), ".2")
test_pipeline.transform(task3, task1, suffix(
".1"), ".3").active_if(lambda: pipeline_active_if)
test_pipeline.collate(task4, [task2, task3], regex(r"(.+)\.[23]"), r"\1.4")
test_pipeline.merge(task5, task4, "test_active_if/summary.5")
class Test_ruffus(unittest.TestCase):
def setUp(self):
try:
shutil.rmtree(tempdir)
except:
pass
os.makedirs(tempdir)
def tearDown(self):
try:
shutil.rmtree(tempdir)
pass
except:
def make_pipeline(state):
'''Build the pipeline by constructing stages and connecting them together'''
# Build an empty pipeline
pipeline = Pipeline(name='crpipe')
# Get a list of paths to all the FASTQ files
fastq_files = state.config.get_option('fastqs')
# Find the path to the reference genome
# Stages are dependent on the state
stages = Stages(state)
# The original FASTQ files
# This is a dummy stage. It is useful because it makes a node in the
# pipeline graph, and gives the pipeline an obvious starting point.
pipeline.originate(task_func=stages.original_fastqs,
name='original_fastqs',
output=fastq_files)
# Convert FASTQ file to FASTA using fastx toolkit
# pipeline.transform(
# task_func=stages.fastq_to_fasta,
# name='fastq_to_fasta',
# input=output_from('original_fastqs'),
# filter=suffix('.fastq.gz'),
# output='.fasta')
# The original reference file
# This is a dummy stage. It is useful because it makes a node in the
# pipeline graph, and gives the pipeline an obvious starting point.
#pipeline.originate(
# task_func=stages.original_reference,
# name='original_reference',
# output=reference_file)
# Run fastQC on the FASTQ files
pipeline.transform(task_func=stages.fastqc,
name='fastqc',
input=output_from('original_fastqs'),
filter=suffix('.fastq.gz'),
output='_fastqc')
# Index the reference using BWA
#pipeline.transform(
# task_func=stages.index_reference_bwa,
# name='index_reference_bwa',
# input=output_from('original_reference'),
# filter=suffix('.fa'),
# output=['.fa.amb', '.fa.ann', '.fa.pac', '.fa.sa', '.fa.bwt'])
# Index the reference using samtools
# pipeline.transform(
# task_func=stages.index_reference_samtools,
# name='index_reference_samtools',
# input=output_from('original_reference'),
# filter=suffix('.fa'),
# output='.fa.fai')
# Index the reference using bowtie 2
# pipeline.transform(
# task_func=stages.index_reference_bowtie2,
# name='index_reference_bowtie2',
# input=output_from('original_reference'),
# filter=formatter('.+/(?P<refname>[a-zA-Z0-9]+\.fa)'),
# output=['{path[0]}/{refname[0]}.1.bt2',
# '{path[0]}/{refname[0]}.2.bt2',
# '{path[0]}/{refname[0]}.3.bt2',
# '{path[0]}/{refname[0]}.4.bt2',
# '{path[0]}/{refname[0]}.rev.1.bt2',
# '{path[0]}/{refname[0]}.rev.2.bt2'],
# extras=['{path[0]}/{refname[0]}'])
# # Create a FASTA sequence dictionary for the reference using picard
# pipeline.transform(
# task_func=stages.reference_dictionary_picard,
# name='reference_dictionary_picard',
# input=output_from('original_reference'),
# filter=suffix('.fa'),
# output='.dict')
# Align paired end reads in FASTQ to the reference producing a BAM file
pipeline.transform(
task_func=stages.align_bwa,
name='align_bwa',
input=output_from('original_fastqs'),
# Match the R1 (read 1) FASTQ file and grab the path and sample name.
# This will be the first input to the stage.
# We assume the sample name may consist of only alphanumeric
# characters.
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+)_R1.fastq.gz'),
# Add two more inputs to the stage:
# 1. The corresponding R2 FASTQ file
add_inputs=add_inputs('{path[0]}/{sample[0]}_R2.fastq.gz'),
# Add an "extra" argument to the state (beyond the inputs and outputs)
# which is the sample name. This is needed within the stage for finding out
# sample specific configuration options
extras=['{sample[0]}'],
# The output file name is the sample name with a .bam extension.
output='{path[0]}/{sample[0]}.bam')
# Sort alignment with sambamba
pipeline.transform(task_func=stages.sort_bam_sambamba,
name='sort_alignment',
input=output_from('align_bwa'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).bam'),
output='{path[0]}/{sample[0]}.sorted.bam')
# Extract MMR genes from the sorted BAM file
pipeline.transform(
task_func=stages.extract_genes_bedtools,
name='extract_genes_bedtools',
input=output_from('sort_alignment'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'),
output='{path[0]}/{sample[0]}.mmr.bam')
# Extract selected chromosomes from the sorted BAM file
pipeline.transform(
task_func=stages.extract_chromosomes_samtools,
name='extract_chromosomes_samtools',
input=output_from('sort_alignment'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'),
output='{path[0]}/{sample[0]}.chroms.bam')
# Index the MMR genes bam file with samtools
pipeline.transform(task_func=stages.index_bam,
name='index_mmr_alignment',
input=output_from('extract_genes_bedtools'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).mmr.bam'),
output='{path[0]}/{sample[0]}.mmr.bam.bai')
# Compute depth of coverage of the alignment with GATK DepthOfCoverage
#pipeline.transform(
# task_func=stages.alignment_coverage_gatk,
# name='alignment_coverage_gatk',
# input=output_from('sort_alignment'),
# filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'),
# add_inputs=add_inputs([reference_file]),
# output='{path[0]}/{sample[0]}.coverage_summary',
# extras=['{path[0]}/{sample[0]}_coverage'])
# Index the alignment with samtools
pipeline.transform(
task_func=stages.index_bam,
name='index_alignment',
input=output_from('sort_alignment'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'),
output='{path[0]}/{sample[0]}.sorted.bam.bai')
# Generate alignment stats with bamtools
pipeline.transform(task_func=stages.bamtools_stats,
name='bamtools_stats',
input=output_from('align_bwa'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).bam'),
output='{path[0]}/{sample[0]}.stats.txt')
# Extract the discordant paired-end alignments
pipeline.transform(task_func=stages.extract_discordant_alignments,
name='extract_discordant_alignments',
input=output_from('align_bwa'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).bam'),
output='{path[0]}/{sample[0]}.discordants.unsorted.bam')
# Extract split-read alignments
pipeline.transform(task_func=stages.extract_split_read_alignments,
name='extract_split_read_alignments',
input=output_from('align_bwa'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).bam'),
output='{path[0]}/{sample[0]}.splitters.unsorted.bam')
# Sort discordant reads.
# Samtools annoyingly takes the prefix of the output bam name as its argument.
# So we pass this as an extra argument. However Ruffus needs to know the full name
# of the output bam file, so we pass that as the normal output parameter.
pipeline.transform(
task_func=stages.sort_bam,
name='sort_discordants',
input=output_from('extract_discordant_alignments'),
filter=formatter(
'.+/(?P<sample>[a-zA-Z0-9]+).discordants.unsorted.bam'),
extras=['{path[0]}/{sample[0]}.discordants'],
output='{path[0]}/{sample[0]}.discordants.bam')
# Index the sorted discordant bam with samtools
# pipeline.transform(
# task_func=stages.index_bam,
# name='index_discordants',
# input=output_from('sort_discordants'),
# filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).discordants.bam'),
# output='{path[0]}/{sample[0]}.discordants.bam.bai')
# Sort discordant reads
# Samtools annoyingly takes the prefix of the output bam name as its argument.
# So we pass this as an extra argument. However Ruffus needs to know the full name
# of the output bam file, so we pass that as the normal output parameter.
pipeline.transform(
task_func=stages.sort_bam,
name='sort_splitters',
input=output_from('extract_split_read_alignments'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).splitters.unsorted.bam'),
extras=['{path[0]}/{sample[0]}.splitters'],
output='{path[0]}/{sample[0]}.splitters.bam')
# Index the sorted splitters bam with samtools
# pipeline.transform(
# task_func=stages.index_bam,
# name='index_splitters',
# input=output_from('sort_splitters'),
# filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).splitters.bam'),
# output='{path[0]}/{sample[0]}.splitters.bam.bai')
# Call structural variants with lumpy
(pipeline.transform(
task_func=stages.structural_variants_lumpy,
name='structural_variants_lumpy',
input=output_from('sort_alignment'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'),
add_inputs=add_inputs([
'{path[0]}/{sample[0]}.splitters.bam',
'{path[0]}/{sample[0]}.discordants.bam'
]),
output='{path[0]}/{sample[0]}.lumpy.vcf').follows('index_alignment').
follows('sort_splitters').follows('sort_discordants'))
# Call genotypes on lumpy output using SVTyper
#(pipeline.transform(
# task_func=stages.genotype_svtyper,
# name='genotype_svtyper',
# input=output_from('structural_variants_lumpy'),
# filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).lumpy.vcf'),
# add_inputs=add_inputs(['{path[0]}/{sample[0]}.sorted.bam', '{path[0]}/{sample[0]}.splitters.bam']),
# output='{path[0]}/{sample[0]}.svtyper.vcf')
# .follows('align_bwa')
# .follows('sort_splitters')
# .follows('index_alignment')
# .follows('index_splitters')
# .follows('index_discordants'))
# Call SVs with Socrates
(pipeline.transform(
task_func=stages.structural_variants_socrates,
name='structural_variants_socrates',
input=output_from('sort_alignment'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'),
# output goes to {path[0]}/socrates/
output=
'{path[0]}/socrates/results_Socrates_paired_{sample[0]}.sorted_long_sc_l25_q5_m5_i95.txt',
extras=['{path[0]}']))
# Call DELs with DELLY
pipeline.merge(task_func=stages.deletions_delly,
name='deletions_delly',
input=output_from('sort_alignment'),
output='delly.DEL.vcf')
# Call DUPs with DELLY
pipeline.merge(task_func=stages.duplications_delly,
name='duplications_delly',
input=output_from('sort_alignment'),
output='delly.DUP.vcf')
# Call INVs with DELLY
pipeline.merge(task_func=stages.inversions_delly,
name='inversions_delly',
input=output_from('sort_alignment'),
output='delly.INV.vcf')
# Call TRAs with DELLY
pipeline.merge(task_func=stages.translocations_delly,
name='translocations_delly',
input=output_from('sort_alignment'),
output='delly.TRA.vcf')
# Join both read pair files using gustaf_mate_joining
#pipeline.transform(
# task_func=stages.gustaf_mate_joining,
# name='gustaf_mate_joining',
# input=output_from('fastq_to_fasta'),
# # Match the R1 (read 1) FASTA file and grab the path and sample name.
# # This will be the first input to the stage.
# # We assume the sample name may consist of only alphanumeric
# # characters.
# filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+)_R1.fasta'),
# # Add one more input to the stage:
# # 1. The corresponding R2 FASTA file
# add_inputs=add_inputs(['{path[0]}/{sample[0]}_R2.fasta']),
# output='{path[0]}/{sample[0]}.joined_mates.fasta')
# Call structural variants with pindel
#(pipeline.transform(
# task_func=stages.structural_variants_pindel,
# name='structural_variants_pindel',
# input=output_from('sort_alignment'),
# filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).sorted.bam'),
# add_inputs=add_inputs(['{path[0]}/{sample[0]}.pindel_config.txt', reference_file]),
# output='{path[0]}/{sample[0]}.pindel')
# .follows('index_reference_bwa')
# .follows('index_reference_samtools'))
return pipeline
def make_pipeline_process(state):
#originate process pipeline state
# Define empty pipeline
pipeline = Pipeline(name='haloplexpipe')
# Get a list of paths to all the directories to be combined for variant calling
run_directories = state.config.get_option('runs')
#grab files from each of the processed directories in "runs"
gatk_files = []
for directory in run_directories:
gatk_files.extend(glob.glob(directory + '/variants/gatk/*.g.vcf'))
stages = Stages(state)
#dummy stage to take the globbed outputs of each run that is to be processed
pipeline.originate(task_func=stages.glob_gatk,
name='glob_gatk',
output=gatk_files)
# Combine G.VCF files for all samples using GATK
pipeline.merge(task_func=stages.combine_gvcf_gatk,
name='combine_gvcf_gatk',
input=output_from('glob_gatk'),
output='processed/gatk/ALL.combined.vcf')
# Genotype G.VCF files using GATK
pipeline.transform(task_func=stages.genotype_gvcf_gatk,
name='genotype_gvcf_gatk',
input=output_from('combine_gvcf_gatk'),
filter=suffix('.combined.vcf'),
output='.raw.vcf')
# Apply GT filters to genotyped vcf
pipeline.transform(task_func=stages.genotype_filter_gatk,
name='genotype_filter_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.raw.vcf'),
output='.raw.gt-filter.vcf')
# Decompose and normalise multiallelic sites
pipeline.transform(task_func=stages.vt_decompose_normalise,
name='vt_decompose_normalise',
input=output_from('genotype_filter_gatk'),
filter=suffix('.raw.gt-filter.vcf'),
output='.raw.gt-filter.decomp.norm.vcf')
# Annotate VCF file using GATK
pipeline.transform(task_func=stages.variant_annotator_gatk,
name='variant_annotator_gatk',
input=output_from('vt_decompose_normalise'),
filter=suffix('.raw.gt-filter.decomp.norm.vcf'),
output='.raw.gt-filter.decomp.norm.annotate.vcf')
# Filter vcf
pipeline.transform(
task_func=stages.gatk_filter,
name='gatk_filter',
input=output_from('variant_annotator_gatk'),
filter=suffix('.raw.gt-filter.decomp.norm.annotate.vcf'),
output='.raw.gt-filter.decomp.norm.annotate.filter.vcf')
#Apply VEP
pipeline.transform(
task_func=stages.apply_vep,
name='apply_vep',
input=output_from('gatk_filter'),
filter=suffix('.raw.gt-filter.decomp.norm.annotate.filter.vcf'),
output='.raw.gt-filter.decomp.norm.annotate.filter.vep.vcf')
####### vardict stuff
vardict_files = []
for directory in run_directories:
vardict_files.extend(
glob.glob(directory + '/variants/vardict/*sorted.vcf.gz'))
#dummy stage to take the globbed outputs of each run that is to be processed
pipeline.originate(task_func=stages.glob_vardict,
name='glob_vardict',
output=vardict_files)
safe_make_dir('processed/vardict')
#concatenate all vardict vcfs
pipeline.merge(task_func=stages.concatenate_vcfs,
name='concatenate_vcfs',
input=output_from('glob_vardict'),
output='processed/vardict/combined.vcf.gz')
pipeline.transform(task_func=stages.vt_decompose_normalise,
name='vt_decompose_normalise_vardict',
input=output_from('concatenate_vcfs'),
filter=suffix('.vcf.gz'),
output='.decomp.norm.vcf.gz')
pipeline.transform(task_func=stages.index_vcfs,
name='index_final_vcf',
input=output_from('vt_decompose_normalise_vardict'),
filter=suffix('.decomp.norm.vcf.gz'),
output='.decomp.norm.vcf.gz.tbi')
(pipeline.transform(
task_func=stages.apply_vep,
name='apply_vep_vardict',
input=output_from('vt_decompose_normalise_vardict'),
filter=suffix('.decomp.norm.vcf.gz'),
output='.decomp.norm.vep.vcf').follows('index_final_vcf'))
return pipeline
"test_active_if/a.2" -> "test_active_if/a.4"
null -> "test_active_if/b.1"
"test_active_if/b.1" -> "test_active_if/b.2"
"test_active_if/b.2" -> "test_active_if/b.4"
"test_active_if/b.4" -> "test_active_if/summary.5"
"""
# alternative syntax
test_pipeline = Pipeline("test")
test_pipeline.originate(task1, ['test_active_if/a.1', 'test_active_if/b.1'], "an extra_parameter")\
.follows(mkdir("test_active_if"))
test_pipeline.transform(task2, task1, suffix(".1"), ".2")
test_pipeline.transform(task3, task1, suffix(".1"),
".3").active_if(lambda: pipeline_active_if)
test_pipeline.collate(task4, [task2, task3], regex(r"(.+)\.[23]"), r"\1.4")
test_pipeline.merge(task5, task4, "test_active_if/summary.5")
class Test_ruffus(unittest.TestCase):
def setUp(self):
try:
shutil.rmtree(tempdir)
except:
pass
os.makedirs(tempdir)
def tearDown(self):
try:
shutil.rmtree(tempdir)
pass
except:
def make_pipeline(state):
'''Build the pipeline by constructing stages and connecting them together'''
# Build an empty pipeline
pipeline = Pipeline(name='vcf_annotation')
# Get a list of paths to all the FASTQ files
vcf_files = state.config.get_option('vcfs')
# Stages are dependent on the state
stages = Stages(state)
# The original VCF files
# This is a dummy stage. It is useful because it makes a node in the
# pipeline graph, and gives the pipeline an obvious starting point.
pipeline.originate(
task_func=stages.original_vcf,
name='original_vcf',
output=vcf_file)
# Decompose VCF using Vt
pipeline.transform(
task_func=stages.decompose_vcf,
name='decompose_vcf',
input=output_from('original_vcf'),
# This will be the first input to the stage.
# We assume the sample name may consist of only alphanumeric
# characters.
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).vcf'),
# Add an "extra" argument to the state (beyond the inputs and outputs)
# which is the VCF file name (e.g. study/family name.
# This is needed within the stage for finding out sample specific
# configuration options
extras=['{sample[0]}'],
# The output file name is the sample name with a .bam extension.
output='{path[0]}/{sample[0]}.decompose.normalize.vcf')
# FILTER COMMON VARIANTS
# ADD FILTER COMMON VARIANTS USING VEP
# Annotate using VEP
pipeline.transform(
task_func=stages.annotate_vep,
name='annotate_vep',
input=output_from('decompose_vcf'),
filter=suffix('.vcf'),
output='.vep.vcf')
# Annotate using SnpEff
pipeline.transform(
task_func=stages.annotate_snpeff,
name='annotate_snpeff',
input=output_from('annotate_vep'),
filter=suffix('.vcf'),
output='.snpeff.vcf')
# Mark duplicates in the BAM file using Picard
pipeline.transform(
task_func=stages.mark_duplicates_picard,
name='mark_duplicates_picard',
input=output_from('sort_bam_picard'),
filter=suffix('.sort.bam'),
# XXX should make metricsup an extra output?
output=['.sort.dedup.bam', '.metricsdup'])
# Generate chromosome intervals using GATK
pipeline.transform(
task_func=stages.chrom_intervals_gatk,
name='chrom_intervals_gatk',
input=output_from('mark_duplicates_picard'),
filter=suffix('.sort.dedup.bam'),
output='.chr.intervals')
# Local realignment using GATK
(pipeline.transform(
task_func=stages.local_realignment_gatk,
name='local_realignment_gatk',
input=output_from('chrom_intervals_gatk'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).chr.intervals'),
add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.bam'),
output='{path[0]}/{sample[0]}.sort.dedup.realn.bam')
.follows('mark_duplicates_picard'))
# Base recalibration using GATK
pipeline.transform(
task_func=stages.base_recalibration_gatk,
name='base_recalibration_gatk',
input=output_from('local_realignment_gatk'),
filter=suffix('.sort.dedup.realn.bam'),
output=['.recal_data.csv', '.count_cov.log'])
# Print reads using GATK
(pipeline.transform(
task_func=stages.print_reads_gatk,
name='print_reads_gatk',
input=output_from('base_recalibration_gatk'),
filter=formatter('.+/(?P<sample>[a-zA-Z0-9]+).recal_data.csv'),
add_inputs=add_inputs('{path[0]}/{sample[0]}.sort.dedup.realn.bam'),
output='{path[0]}/{sample[0]}.sort.dedup.realn.recal.bam')
.follows('local_realignment_gatk'))
# Call variants using GATK
pipeline.transform(
task_func=stages.call_variants_gatk,
name='call_variants_gatk',
input=output_from('print_reads_gatk'),
filter=suffix('.sort.dedup.realn.recal.bam'),
output='.raw.snps.indels.g.vcf')
# Combine G.VCF files for all samples using GATK
pipeline.merge(
task_func=stages.combine_gvcf_gatk,
name='combine_gvcf_gatk',
input=output_from('call_variants_gatk'),
output='PCExomes.mergegvcf.vcf')
# Genotype G.VCF files using GATK
pipeline.transform(
task_func=stages.genotype_gvcf_gatk,
name='genotype_gvcf_gatk',
input=output_from('combine_gvcf_gatk'),
filter=suffix('.mergegvcf.vcf'),
output='.genotyped.vcf')
# SNP recalibration using GATK
pipeline.transform(
task_func=stages.snp_recalibrate_gatk,
name='snp_recalibrate_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.genotyped.vcf'),
output=['.snp_recal', '.snp_tranches', '.snp_plots.R'])
# INDEL recalibration using GATK
pipeline.transform(
task_func=stages.indel_recalibrate_gatk,
name='indel_recalibrate_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.genotyped.vcf'),
output=['.indel_recal', '.indel_tranches', '.indel_plots.R'])
# Apply SNP recalibration using GATK
(pipeline.transform(
task_func=stages.apply_snp_recalibrate_gatk,
name='apply_snp_recalibrate_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.genotyped.vcf'),
add_inputs=add_inputs(['PCExomes.snp_recal', 'PCExomes.snp_tranches']),
output='.recal_SNP.vcf')
.follows('snp_recalibrate_gatk'))
# Apply INDEL recalibration using GATK
(pipeline.transform(
task_func=stages.apply_indel_recalibrate_gatk,
name='apply_indel_recalibrate_gatk',
input=output_from('genotype_gvcf_gatk'),
filter=suffix('.genotyped.vcf'),
add_inputs=add_inputs(
['PCExomes.indel_recal', 'PCExomes.indel_tranches']),
output='.recal_INDEL.vcf')
.follows('indel_recalibrate_gatk'))
# Combine variants using GATK
(pipeline.transform(
task_func=stages.combine_variants_gatk,
name='combine_variants_gatk',
input=output_from('apply_snp_recalibrate_gatk'),
filter=suffix('.recal_SNP.vcf'),
add_inputs=add_inputs(['PCExomes.recal_INDEL.vcf']),
output='.combined.vcf')
.follows('apply_indel_recalibrate_gatk'))
# Select variants using GATK
pipeline.transform(
task_func=stages.select_variants_gatk,
name='select_variants_gatk',
input=output_from('combine_variants_gatk'),
filter=suffix('.combined.vcf'),
output='.selected.vcf')
return pipeline
