def process_subpipeline(**connections):
w = WorkflowBuilder("somatic_subpipeline")
w.input("reference", FastaWithDict)
w.input("reads", Array(FastqGzPair))
w.input("cutadapt_adapters", File(optional=True))
w.input("sample_name", String)
w.step("fastqc", FastQC_0_11_5(reads=w.reads), scatter="reads")
w.step(
"getfastqc_adapters",
ParseFastqcAdaptors(
fastqc_datafiles=w.fastqc.datafile,
cutadapt_adaptors_lookup=w.cutadapt_adapters,
),
scatter="fastqc_datafiles",
)
w.step(
"align_and_sort",
BwaAligner(
fastq=w.reads,
reference=w.reference,
sample_name=w.sample_name,
sortsam_tmpDir=".",
cutadapt_adapter=w.getfastqc_adapters,
cutadapt_removeMiddle3Adapter=w.getfastqc_adapters,
),
scatter=[
"fastq", "cutadapt_adapter", "cutadapt_removeMiddle3Adapter"
],
)
w.step(
"merge_and_mark",
MergeAndMarkBams_4_1_3(bams=w.align_and_sort.out,
sampleName=w.sample_name),
)
w.output("out", source=w.merge_and_mark.out)
w.output("reports",
source=w.fastqc.out,
output_folder=[w.sample_name, "reports"])
return w(**connections)
def add_align(self):
self.step(
"getfastqc_adapters",
ParseFastqcAdaptors(
fastqc_datafiles=self.fastqc.datafile,
cutadapt_adaptors_lookup=self.cutadapt_adapters,
),
scatter="fastqc_datafiles",
)
self.step(
"align_and_sort",
BwaAligner(
fastq=self.fastqs,
reference=self.reference,
sample_name=self.sample_name,
sortsam_tmpDir="./tmp",
cutadapt_adapter=self.getfastqc_adapters,
cutadapt_removeMiddle3Adapter=self.getfastqc_adapters,
),
scatter=[
"fastq", "cutadapt_adapter", "cutadapt_removeMiddle3Adapter"
],
)
def constructor(self):
# Inputs
self.input("sample_name", String)
self.input("fastqs", Array(FastqGzPair))
self.input("reference", FastaWithDict)
self.input("region_bed", Bed)
self.input("region_bed_extended", Bed)
self.input("region_bed_annotated", Bed)
self.input("genecoverage_bed", Bed)
self.input("genome_file", TextFile)
self.input("black_list", Bed(optional=True))
self.input("snps_dbsnp", VcfTabix)
self.input("snps_1000gp", VcfTabix)
self.input("known_indels", VcfTabix)
self.input("mills_indels", VcfTabix)
# fastqc
self.step(
"fastqc", FastQC_0_11_5(reads=self.fastqs, threads=4), scatter="reads"
)
# get the overrepresentative sequence from fastqc
self.step(
"getfastqc_adapters",
ParseFastqcAdaptors(fastqc_datafiles=self.fastqc.datafile,),
scatter="fastqc_datafiles",
)
# align and generate sorted index bam
self.step(
"align_and_sort",
BwaAligner(
fastq=self.fastqs,
reference=self.reference,
sample_name=self.sample_name,
sortsam_tmpDir=".",
cutadapt_adapter=self.getfastqc_adapters,
cutadapt_removeMiddle3Adapter=self.getfastqc_adapters,
),
scatter=["fastq", "cutadapt_adapter", "cutadapt_removeMiddle3Adapter"],
)
# merge into one bam and markdups
self.step(
"merge_and_mark",
MergeAndMarkBams_4_1_3(
bams=self.align_and_sort.out, sampleName=self.sample_name
),
)
# performance: doc
self.step(
"annotate_doc",
AnnotateDepthOfCoverage_0_1_0(
bam=self.merge_and_mark.out,
bed=self.region_bed_annotated,
reference=self.reference,
sample_name=self.sample_name,
),
)
# performance
self.step(
"performance_summary",
PerformanceSummaryTargeted_0_1_0(
bam=self.merge_and_mark.out,
region_bed=self.region_bed,
genecoverage_bed=self.genecoverage_bed,
sample_name=self.sample_name,
genome_file=self.genome_file,
),
)
# gridss
self.step(
"gridss",
Gridss_2_6_2(
bams=self.merge_and_mark.out,
reference=self.reference,
blacklist=self.black_list,
tmpdir=".",
),
)
# post gridss r script here
# self.step("gridss_post_r", )
# gatk bqsr bam
self.step(
"bqsr",
GATKBaseRecalBQSRWorkflow_4_1_3(
bam=self.merge_and_mark.out,
intervals=self.region_bed_extended,
reference=self.reference,
snps_dbsnp=self.snps_dbsnp,
snps_1000gp=self.snps_1000gp,
known_indels=self.known_indels,
mills_indels=self.mills_indels,
),
)
# haploytype caller
self.step(
"haplotype_caller",
Gatk4HaplotypeCaller_4_1_3(
inputRead=self.bqsr.out,
intervals=self.region_bed_extended,
reference=self.reference,
dbsnp=self.snps_dbsnp,
pairHmmImplementation="LOGLESS_CACHING",
),
)
self.step(
"splitnormalisevcf",
SplitMultiAlleleNormaliseVcf(
compressedVcf=self.haplotype_caller.out, reference=self.reference
),
)
self.step(
"addbamstats",
AddBamStatsGermline_0_1_0(
bam=self.merge_and_mark.out,
vcf=self.splitnormalisevcf.out,
reference=self.reference,
),
)
# output
self.output("fastq_qc", source=self.fastqc.out, output_folder="QC")
self.output("markdups_bam", source=self.merge_and_mark.out, output_folder="BAM")
self.output("doc", source=self.annotate_doc.out, output_folder="PERFORMANCE")
self.output(
"summary", source=self.performance_summary.out, output_folder="PERFORMANCE"
)
self.output(
"gene_summary",
source=self.performance_summary.geneFileOut,
output_folder="PERFORMANCE",
)
self.output(
"region_summary",
source=self.performance_summary.regionFileOut,
output_folder="PERFORMANCE",
)
self.output("gridss_vcf", source=self.gridss.out, output_folder="SV")
self.output("gridss_bam", source=self.gridss.assembly, output_folder="SV")
self.output("hap_vcf", source=self.haplotype_caller.out, output_folder="VCF")
self.output("hap_bam", source=self.haplotype_caller.bam, output_folder="VCF")
self.output("normalise_vcf", source=self.addbamstats.out, output_folder="VCF")
def constructor(self):
# Inputs
self.input("sample_name", String)
self.input("fastqs", Array(FastqGzPair))
self.input("seqrun", String, doc="SeqRun Name (for Vcf2Tsv)")
self.input("reference", FastaWithDict)
self.input("region_bed", Bed)
self.input("region_bed_extended", Bed)
self.input("region_bed_annotated", Bed)
self.input("genecoverage_bed", Bed)
self.input("genome_file", TextFile)
self.input("panel_name", String)
self.input("vcfcols", TextFile)
self.input("black_list", Bed(optional=True))
self.input("snps_dbsnp", VcfTabix)
self.input("snps_1000gp", VcfTabix)
self.input("known_indels", VcfTabix)
self.input("mills_indels", VcfTabix)
self.input("mutalyzer_server", String)
self.input("pathos_db", String)
self.input("maxRecordsInRam", Int)
# tumor only
self.input("gnomad", VcfTabix)
self.input("panel_of_normals", VcfTabix(optional=True))
# fastqc
self.step(
"fastqc", FastQC_0_11_5(reads=self.fastqs, threads=4), scatter="reads"
)
# get the overrepresentative sequence from fastqc
self.step(
"getfastqc_adapters",
ParseFastqcAdaptors(fastqc_datafiles=self.fastqc.datafile,),
scatter="fastqc_datafiles",
)
# align and generate sorted index bam
self.step(
"align_and_sort",
BwaAligner(
fastq=self.fastqs,
reference=self.reference,
sample_name=self.sample_name,
sortsam_tmpDir=".",
cutadapt_adapter=self.getfastqc_adapters,
cutadapt_removeMiddle3Adapter=self.getfastqc_adapters,
),
scatter=["fastq", "cutadapt_adapter", "cutadapt_removeMiddle3Adapter"],
)
# merge into one bam and markdups
self.step(
"merge_and_mark",
MergeAndMarkBams_4_1_3(
bams=self.align_and_sort.out,
sampleName=self.sample_name,
maxRecordsInRam=self.maxRecordsInRam,
),
)
# performance: doc
self.step(
"annotate_doc",
AnnotateDepthOfCoverage_0_1_0(
bam=self.merge_and_mark.out,
bed=self.region_bed_annotated,
reference=self.reference,
sample_name=self.sample_name,
),
)
# performance
self.step(
"performance_summary",
PerformanceSummaryTargeted_0_1_0(
bam=self.merge_and_mark.out,
region_bed=self.region_bed,
genecoverage_bed=self.genecoverage_bed,
sample_name=self.sample_name,
genome_file=self.genome_file,
),
)
# gridss
self.step(
"gridss",
Gridss_2_6_2(
bams=self.merge_and_mark.out,
reference=self.reference,
blacklist=self.black_list,
tmpdir=".",
),
)
# post gridss r for tumor only + tumor only mode
# self.step("gridss_post_r", GRIDSSProcessOutput(inp=self.gridss.out))
# gatk bqsr bam
self.step(
"bqsr",
GATKBaseRecalBQSRWorkflow_4_1_3(
bam=self.merge_and_mark.out,
intervals=self.region_bed_extended,
reference=self.reference,
snps_dbsnp=self.snps_dbsnp,
snps_1000gp=self.snps_1000gp,
known_indels=self.known_indels,
mills_indels=self.mills_indels,
),
)
# mutect2
self.step(
"mutect2",
GatkSomaticVariantCallerTumorOnlyTargeted(
bam=self.bqsr.out,
intervals=self.region_bed_extended,
reference=self.reference,
gnomad=self.gnomad,
panel_of_normals=self.panel_of_normals,
),
)
# haplotypecaller to do: take base recal away from the
self.step(
"haplotype_caller",
Gatk4HaplotypeCaller_4_1_3(
inputRead=self.bqsr.out,
intervals=self.region_bed_extended,
reference=self.reference,
dbsnp=self.snps_dbsnp,
pairHmmImplementation="LOGLESS_CACHING",
),
)
self.step(
"splitnormalisevcf",
SplitMultiAlleleNormaliseVcf(
compressedVcf=self.haplotype_caller.out, reference=self.reference
),
)
# combine variants
self.step(
"combinevariants",
CombineVariants_0_0_8(
vcfs=[self.splitnormalisevcf.out, self.mutect2.out],
type="germline",
columns=["AD", "DP", "AF", "GT"],
),
)
self.step("compressvcf", BGZip_1_9(file=self.combinevariants.out))
self.step("sortvcf", BcfToolsSort_1_9(vcf=self.compressvcf.out))
self.step("uncompressvcf", UncompressArchive(file=self.sortvcf.out))
# addbamstats
self.step(
"addbamstats",
AddBamStatsGermline_0_1_0(
bam=self.merge_and_mark.out,
vcf=self.uncompressvcf.out,
reference=self.reference,
),
)
# Molpath specific processes
self.step("compressvcf2", BGZip_1_9(file=self.addbamstats.out))
self.step("tabixvcf", TabixLatest(inp=self.compressvcf2.out))
self.step(
"calculate_variant_length",
VcfLength_1_0_1(vcf=self.tabixvcf.out),
doc="Add the length column for the output of AddBamStats",
)
filter_for_variants = self.input("filter_for_vcfs", str, default="length > 150")
self.step(
"filter_variants_1_failed",
VcfFilter_1_0_1(
vcf=self.calculate_variant_length.out, info_filter=filter_for_variants
),
)
self.step(
"filter_variants_1",
VcfFilter_1_0_1(
vcf=self.calculate_variant_length.out,
info_filter=filter_for_variants,
invert=True, # -v param
),
)
# Jiaan: copy over from the FRCP, can take the block comment out
# # This one is the in-house molpath step
# self.step(
# "normalise_vcfs",
# NormaliseVcf_1_5_4(
# pathos_version=self.pathos_db,
# mutalyzer=self.mutalyzer_server, # mutalyzer="https://vmpr-res-mutalyzer1.unix.petermac.org.au",
# rdb=self.pathos_db, # rdb="pa_uat",
# inp=self.filter_variants_1.out,
# ),
# )
# # repeat remove 150bp variants (workaround for normalise_vcf bug)
# self.step(
# "filter_variants_2_failed",
# VcfFilter_1_0_1(
# vcf=self.normalise_vcfs.out, info_filter=filter_for_variants
# ),
# )
# self.step(
# "filter_variants_2",
# VcfFilter_1_0_1(
# vcf=self.normalise_vcfs.out,
# info_filter=filter_for_variants,
# invert=True, # -v param
# ),
# )
# self.step(
# "convert_to_tsv",
# Vcf2Tsv_1_5_4(
# pathos_version=self.pathos_db,
# inp=self.filter_variants_2.out,
# sample=self.sample_name,
# columns=self.vcfcols,
# seqrun=self.seqrun,
# ),
# )
# self.step(
# "index_with_igvtools", IgvIndexFeature_2_5_3(inp=self.filter_variants_2.out)
# )
# output
self.output("fastq_qc", source=self.fastqc.out, output_folder="QC")
self.output("markdups_bam", source=self.merge_and_mark.out, output_folder="BAM")
self.output(
"doc_out", source=self.annotate_doc.out, output_folder="PERFORMANCE"
)
self.output(
"summary", source=self.performance_summary.out, output_folder="PERFORMANCE"
)
self.output(
"gene_summary",
source=self.performance_summary.geneFileOut,
output_folder="PERFORMANCE",
)
self.output(
"region_summary",
source=self.performance_summary.regionFileOut,
output_folder="PERFORMANCE",
)
self.output("gridss_vcf", source=self.gridss.out, output_folder="SV")
self.output("gridss_bam", source=self.gridss.assembly, output_folder="SV")
self.output(
"haplotypecaller_vcf",
source=self.haplotype_caller.out,
output_folder="VCF",
)
self.output(
"haplotypecaller_bam",
source=self.haplotype_caller.bam,
output_folder="VCF",
)
self.output(
"haplotypecaller_norm",
source=self.splitnormalisevcf.out,
output_folder="VCF",
)
self.output("mutect2_vcf", source=self.mutect2.variants, output_folder="VCF")
self.output("mutect2_bam", source=self.mutect2.out_bam, output_folder="VCF")
self.output("mutect2_norm", source=self.mutect2.out, output_folder="VCF")
self.output("addbamstats_vcf", source=self.addbamstats.out)
def process_subpipeline(**connections):
w = WorkflowBuilder("somatic_subpipeline")
# INPUTS
w.input("reads", Array(FastqGzPair))
w.input("sample_name", String)
w.input("reference", FastaWithDict)
w.input("cutadapt_adapters", File(optional=True))
w.input("gatk_intervals", Array(Bed))
w.input("snps_dbsnp", VcfTabix)
w.input("snps_1000gp", VcfTabix)
w.input("known_indels", VcfTabix)
w.input("mills_indels", VcfTabix)
# STEPS
w.step("fastqc", FastQC_0_11_8(reads=w.reads), scatter="reads")
w.step(
"getfastqc_adapters",
ParseFastqcAdaptors(
fastqc_datafiles=w.fastqc.datafile,
cutadapt_adaptors_lookup=w.cutadapt_adapters,
),
scatter="fastqc_datafiles",
)
w.step(
"align_and_sort",
BwaAligner(
fastq=w.reads,
reference=w.reference,
sample_name=w.sample_name,
sortsam_tmpDir=None,
cutadapt_adapter=w.getfastqc_adapters,
cutadapt_removeMiddle3Adapter=w.getfastqc_adapters,
),
scatter=[
"fastq", "cutadapt_adapter", "cutadapt_removeMiddle3Adapter"
],
)
w.step(
"merge_and_mark",
MergeAndMarkBams_4_1_3(bams=w.align_and_sort.out,
sampleName=w.sample_name),
)
# Temporarily remove GATK4 DepthOfCoverage for performance reasons, see:
# https://gatk.broadinstitute.org/hc/en-us/community/posts/360071895391-Speeding-up-GATK4-DepthOfCoverage
# w.step(
# "coverage",
# Gatk4DepthOfCoverage_4_1_6(
# bam=w.merge_and_mark.out,
# reference=w.reference,
# intervals=w.gatk_intervals,
# omitDepthOutputAtEachBase=True,
# # countType="COUNT_FRAGMENTS_REQUIRE_SAME_BASE",
# summaryCoverageThreshold=[1, 50, 100, 300, 500],
# outputPrefix=w.sample_name,
# ),
# )
w.step(
"calculate_performancesummary_genomefile",
GenerateGenomeFileForBedtoolsCoverage(reference=w.reference),
)
w.step(
"performance_summary",
PerformanceSummaryGenome_0_1_0(
bam=w.merge_and_mark.out,
sample_name=w.sample_name,
genome_file=w.calculate_performancesummary_genomefile.out,
),
)
# OUTPUTS
w.output("out_bam", source=w.merge_and_mark.out)
w.output("out_fastqc_reports", source=w.fastqc.out)
# w.output("depth_of_coverage", source=w.coverage.out_sampleSummary)
w.output(
"out_performance_summary",
source=w.performance_summary.performanceSummaryOut,
)
return w(**connections)
def constructor(self):
self.input(
"sample_name",
String,
doc=InputDocumentation(
"Sample name from which to generate the readGroupHeaderLine for BwaMem",
quality=InputQualityType.user,
example="NA12878",
),
)
self.input(
"fastqs",
Array(FastqGzPair),
doc=InputDocumentation(
"An array of FastqGz pairs. These are aligned separately and merged "
"to create higher depth coverages from multiple sets of reads",
quality=InputQualityType.user,
example="[[BRCA1_R1.fastq.gz, BRCA1_R2.fastq.gz]]",
),
)
self.input(
"reference",
FastaWithDict,
doc=InputDocumentation(
"""\
The reference genome from which to align the reads. This requires a number indexes (can be generated \
with the 'IndexFasta' pipeline This pipeline has been tested using the HG38 reference set.
This pipeline expects the assembly references to be as they appear in the GCP example:
- (".fai", ".amb", ".ann", ".bwt", ".pac", ".sa", "^.dict").""",
quality=InputQualityType.static,
example=
"HG38: https://console.cloud.google.com/storage/browser/genomics-public-data/references/hg38/v0/\n\n"
"File: gs://genomics-public-data/references/hg38/v0/Homo_sapiens_assembly38.fasta",
),
)
self.input(
"cutadapt_adapters",
File(optional=True),
doc=InputDocumentation(
"Specifies a containment list for cutadapt, which contains a list of sequences to determine valid overrepresented sequences from "
"the FastQC report to trim with Cuatadapt. The file must contain sets of named adapters in the form: "
"``name[tab]sequence``. Lines prefixed with a hash will be ignored.",
quality=InputQualityType.static,
example=
"https://github.com/csf-ngs/fastqc/blob/master/Contaminants/contaminant_list.txt",
),
)
self.input(
"gatk_intervals",
Array(Bed),
doc=InputDocumentation(
"List of intervals over which to split the GATK variant calling",
quality=InputQualityType.static,
example="BRCA1.bed",
),
)
self.input(
"snps_dbsnp",
VcfTabix,
doc=InputDocumentation(
"From the GATK resource bundle, passed to BaseRecalibrator as ``known_sites``",
quality=InputQualityType.static,
example=
"HG38: https://console.cloud.google.com/storage/browser/genomics-public-data/references/hg38/v0/\n\n"
"(WARNING: The file available from the genomics-public-data resource on Google Cloud Storage is NOT compressed and indexed. This will need to be completed prior to starting the pipeline.\n\n"
"File: gs://genomics-public-data/references/hg38/v0/Homo_sapiens_assembly38.dbsnp138.vcf.gz",
),
)
self.input(
"snps_1000gp",
VcfTabix,
doc=InputDocumentation(
"From the GATK resource bundle, passed to BaseRecalibrator as ``known_sites``",
quality=InputQualityType.static,
example=
"HG38: https://console.cloud.google.com/storage/browser/genomics-public-data/references/hg38/v0/\n\n"
"File: gs://genomics-public-data/references/hg38/v0/1000G_phase1.snps.high_confidence.hg38.vcf.gz",
),
)
self.input(
"known_indels",
VcfTabix,
doc=InputDocumentation(
"From the GATK resource bundle, passed to BaseRecalibrator as ``known_sites``",
quality=InputQualityType.static,
example=
"HG38: https://console.cloud.google.com/storage/browser/genomics-public-data/references/hg38/v0/\n\n"
"File: gs://genomics-public-data/references/hg38/v0/Homo_sapiens_assembly38.known_indels.vcf.gz",
),
)
self.input(
"mills_indels",
VcfTabix,
doc=InputDocumentation(
"From the GATK resource bundle, passed to BaseRecalibrator as ``known_sites``",
quality=InputQualityType.static,
example=
"HG38: https://console.cloud.google.com/storage/browser/genomics-public-data/references/hg38/v0/\n\n"
"File: gs://genomics-public-data/references/hg38/v0/Mills_and_1000G_gold_standard.indels.hg38.vcf.gz",
),
)
# STEPS
self.step("fastqc", FastQC_0_11_5(reads=self.fastqs), scatter="reads")
self.step(
"getfastqc_adapters",
ParseFastqcAdaptors(
fastqc_datafiles=self.fastqc.datafile,
cutadapt_adaptors_lookup=self.cutadapt_adapters,
),
scatter="fastqc_datafiles",
# when=NotNullOperator(self.cutadapt_adapters)
)
self.step(
"align_and_sort",
BwaAligner(
fastq=self.fastqs,
reference=self.reference,
sample_name=self.sample_name,
sortsam_tmpDir=".",
cutadapt_adapter=self.getfastqc_adapters,
cutadapt_removeMiddle3Adapter=self.getfastqc_adapters,
),
scatter=[
"fastq", "cutadapt_adapter", "cutadapt_removeMiddle3Adapter"
],
)
self.step(
"merge_and_mark",
MergeAndMarkBams_4_1_3(bams=self.align_and_sort,
sampleName=self.sample_name),
)
# VARIANT CALLERS
# GATK
self.step(
"vc_gatk",
GatkGermlineVariantCaller_4_1_3(
bam=self.merge_and_mark,
intervals=self.gatk_intervals,
reference=self.reference,
snps_dbsnp=self.snps_dbsnp,
snps_1000gp=self.snps_1000gp,
known_indels=self.known_indels,
mills_indels=self.mills_indels,
),
scatter="intervals",
)
self.step("vc_gatk_merge", Gatk4GatherVcfs_4_0(vcfs=self.vc_gatk.out))
# sort
self.step("sort_combined",
BcfToolsSort_1_9(vcf=self.vc_gatk_merge.out))
self.output(
"bam",
source=self.merge_and_mark.out,
output_folder=["bams", self.sample_name],
output_name=self.sample_name,
doc="Aligned and indexed bam.",
)
self.output(
"reports",
source=self.fastqc.out,
output_folder=["reports", self.sample_name],
doc="A zip file of the FastQC quality report.",
)
self.output(
"variants",
source=self.sort_combined.out,
output_folder="variants",
output_name=self.sample_name,
doc="Merged variants from the GATK caller",
)
self.output(
"variants_split",
source=self.vc_gatk.out,
output_folder=["variants", "byInterval"],
doc="Unmerged variants from the GATK caller (by interval)",
)