def merge_and_cov_scaffolds(self):
jobs = []
for sample in self.samples:
cov_directory = os.path.join("scaffolds", sample.name, "ray", "ray" + config.param('ray', 'kmer'), "cov")
ray_directory = os.path.join("scaffolds", sample.name, "ray", "ray" + config.param('ray', 'kmer'))
job = picard.merge_sam_files(
[
os.path.join(cov_directory, "sclip.1.bam"),
os.path.join(cov_directory, "sclip.2.bam"),
os.path.join(cov_directory, "OEAUNMAP.1.bam"),
os.path.join(cov_directory, "OEAUNMAP.2.bam"),
os.path.join(cov_directory, "ORPHAN.bam"),
],
os.path.join(cov_directory, "readunmap.bam")
)
job.name = "covSca_merge_" + sample.name
jobs.append(job)
job = bvatools.depth_of_coverage(
os.path.join(cov_directory, "readunmap.bam"),
os.path.join(cov_directory, "readunmap.cov.txt"),
[],
os.path.join(ray_directory, "Scaffolds.fasta"),
"--gc --ommitN --minMappingQuality " + config.param('DEFAULT', 'min_mapping_quality') + " --threads " + config.param('merge_and_cov_scaffolds', 'threads')
)
job.name = "covSca_" + sample.name
jobs.append(job)
return jobs
def get_metrics_jobs(self, readset):
jobs = []
input_file_prefix = readset.bam + '.'
input = input_file_prefix + "bam"
job = picard.collect_multiple_metrics(input, input_file_prefix + "metrics",
reference_sequence=readset.reference_file)
job.name = "picard_collect_multiple_metrics." + readset.name + ".met" + "." + readset.run + "." + readset.lane
jobs.append(job)
if readset.beds:
coverage_bed = readset.beds[0]
full_coverage_bed = (self.output_dir + os.sep + coverage_bed)
else:
coverage_bed = None
full_coverage_bed = None
if coverage_bed:
if (not os.path.exists(full_coverage_bed)) and \
(coverage_bed not in BwaRunProcessingAligner.downloaded_bed_files):
# Download the bed file
command = config.param('DEFAULT', 'fetch_bed_file_command').format(
output_directory=self.output_dir,
filename=coverage_bed
)
job = Job([], [full_coverage_bed], command=command, name="bed_download." + coverage_bed)
BwaRunProcessingAligner.downloaded_bed_files.append(coverage_bed)
jobs.append(job)
interval_list = re.sub("\.[^.]+$", ".interval_list", coverage_bed)
if interval_list not in BwaRunProcessingAligner.created_interval_lists:
# Create one job to generate the interval list from the bed file
ref_dict = os.path.splitext(readset.reference_file)[0] + '.dict'
job = tools.bed2interval_list(ref_dict, full_coverage_bed, interval_list)
job.name = "interval_list." + coverage_bed
BwaRunProcessingAligner.created_interval_lists.append(interval_list)
jobs.append(job)
job = picard.calculate_hs_metrics(input_file_prefix + "bam", input_file_prefix + "metrics.onTarget.txt",
interval_list, reference_sequence=readset.reference_file)
job.name = "picard_calculate_hs_metrics." + readset.name + ".hs" + "." + readset.run + "." + readset.lane
jobs.append(job)
jobs.extend(self.verify_bam_id(readset, full_coverage_bed))
job = bvatools.depth_of_coverage(
input,
input_file_prefix + "metrics.targetCoverage.txt",
full_coverage_bed,
other_options=config.param('bvatools_depth_of_coverage', 'other_options', required=False),
reference_genome=readset.reference_file
)
job.name = "bvatools_depth_of_coverage." + readset.name + ".doc" + "." + readset.run + "." + readset.lane
jobs.append(job)
return jobs
def metrics(self):
"""
Compute metrics and generate coverage tracks per sample. Multiple metrics are computed at this stage:
Number of raw reads, Number of filtered reads, Number of aligned reads, Number of duplicate reads,
Median, mean and standard deviation of insert sizes of reads after alignment, percentage of bases
covered at X reads (%_bases_above_50 means the % of exons bases which have at least 50 reads)
whole genome or targeted percentage of bases covered at X reads (%_bases_above_50 means the % of exons
bases which have at least 50 reads). A TDF (.tdf) coverage track is also generated at this step
for easy visualization of coverage in the IGV browser.
"""
jobs = []
for sample in self.samples:
input_file_prefix = os.path.join(
"alignment", sample.name, sample.name + ".matefixed.sorted.")
input = input_file_prefix + "bam"
job = picard.collect_multiple_metrics(
input, input_file_prefix + "all.metrics")
job.name = "picard_collect_multiple_metrics." + sample.name
job.samples = [sample]
jobs.append(job)
# Compute genome or target coverage with BVATools
job = bvatools.depth_of_coverage(
input,
input_file_prefix + "coverage.tsv",
bvatools.resolve_readset_coverage_bed(sample.readsets[0]),
other_options=config.param('bvatools_depth_of_coverage',
'other_options',
required=False))
job.name = "bvatools_depth_of_coverage." + sample.name
job.samples = [sample]
jobs.append(job)
job = igvtools.compute_tdf(input, input + ".tdf")
job.name = "igvtools_compute_tdf." + sample.name
job.samples = [sample]
jobs.append(job)
return jobs
def metrics(self):
"""
Compute metrics and generate coverage tracks per sample. Multiple metrics are computed at this stage:
Number of raw reads, Number of filtered reads, Number of aligned reads, Number of duplicate reads,
Median, mean and standard deviation of insert sizes of reads after alignment, percentage of bases
covered at X reads (%_bases_above_50 means the % of exons bases which have at least 50 reads)
whole genome or targeted percentage of bases covered at X reads (%_bases_above_50 means the % of exons
bases which have at least 50 reads). A TDF (.tdf) coverage track is also generated at this step
for easy visualization of coverage in the IGV browser.
"""
jobs = []
for sample in self.samples:
input_file_prefix = os.path.join("alignment", sample.name, sample.name + ".matefixed.sorted.")
input = input_file_prefix + "bam"
job = picard.collect_multiple_metrics(input, input_file_prefix + "all.metrics")
job.name = "picard_collect_multiple_metrics." + sample.name
jobs.append(job)
# Compute genome or target coverage with BVATools
job = bvatools.depth_of_coverage(
input,
input_file_prefix + "coverage.tsv",
bvatools.resolve_readset_coverage_bed(sample.readsets[0]),
other_options=config.param('bvatools_depth_of_coverage', 'other_options', required=False)
)
job.name = "bvatools_depth_of_coverage." + sample.name
jobs.append(job)
job = igvtools.compute_tdf(input, input + ".tdf")
job.name = "igvtools_compute_tdf." + sample.name
jobs.append(job)
return jobs
def metrics(self):
"""
Compute metrics and generate coverage tracks per sample. Multiple metrics are computed at this stage:
Number of raw reads, Number of filtered reads, Number of aligned reads, Number of duplicate reads,
Median, mean and standard deviation of insert sizes of reads after alignment, percentage of bases
covered at X reads (%_bases_above_50 means the % of exons bases which have at least 50 reads)
whole genome or targeted percentage of bases covered at X reads (%_bases_above_50 means the % of exons
bases which have at least 50 reads). A TDF (.tdf) coverage track is also generated at this step
for easy visualization of coverage in the IGV browser.
"""
# check the library status
library, bam = {}, {}
for readset in self.readsets:
if not library.has_key(readset.sample):
library[readset.sample] = "SINGLE_END"
if readset.run_type == "PAIRED_END":
library[readset.sample] = "PAIRED_END"
if not bam.has_key(readset.sample):
bam[readset.sample] = ""
if readset.bam:
bam[readset.sample] = readset.bam
jobs = []
created_interval_lists = []
for sample in self.samples:
file_prefix = os.path.join("alignment", sample.name,
sample.name + ".sorted.dedup.")
coverage_bed = bvatools.resolve_readset_coverage_bed(
sample.readsets[0])
candidate_input_files = [[file_prefix + "bam"]]
if bam[sample]:
candidate_input_files.append([bam[sample]])
[input] = self.select_input_files(candidate_input_files)
job = picard.collect_multiple_metrics(input,
re.sub(
"bam", "all.metrics",
input),
library_type=library[sample])
job.name = "picard_collect_multiple_metrics." + sample.name
job.samples = [sample]
jobs.append(job)
# Compute genome coverage with GATK
job = gatk.depth_of_coverage(input,
re.sub("bam", "all.coverage", input),
coverage_bed)
job.name = "gatk_depth_of_coverage.genome." + sample.name
job.samples = [sample]
jobs.append(job)
# Compute genome or target coverage with BVATools
job = bvatools.depth_of_coverage(
input,
re.sub("bam", "coverage.tsv", input),
coverage_bed,
other_options=config.param('bvatools_depth_of_coverage',
'other_options',
required=False))
job.name = "bvatools_depth_of_coverage." + sample.name
job.samples = [sample]
jobs.append(job)
if coverage_bed:
# Get on-target reads (if on-target context is detected)
ontarget_bam = re.sub("bam", "ontarget.bam", input)
flagstat_output = re.sub("bam", "bam.flagstat", ontarget_bam)
job = concat_jobs([
bedtools.intersect(input, ontarget_bam, coverage_bed),
samtools.flagstat(ontarget_bam, flagstat_output)
])
job.name = "ontarget_reads." + sample.name
job.removable_files = [ontarget_bam]
job.samples = [sample]
jobs.append(job)
# Compute on target percent of hybridisation based capture
interval_list = re.sub("\.[^.]+$", ".interval_list",
coverage_bed)
if not interval_list in created_interval_lists:
job = tools.bed2interval_list(None, coverage_bed,
interval_list)
job.name = "interval_list." + os.path.basename(
coverage_bed)
jobs.append(job)
created_interval_lists.append(interval_list)
file_prefix = os.path.join("alignment", sample.name,
sample.name + ".sorted.dedup.")
job = picard.calculate_hs_metrics(file_prefix + "bam",
file_prefix + "onTarget.tsv",
interval_list)
job.name = "picard_calculate_hs_metrics." + sample.name
job.samples = [sample]
jobs.append(job)
# Calculate the number of reads with higher mapping quality than the threshold passed in the ini file
job = concat_jobs([
samtools.view(
input, re.sub(".bam", ".filtered_reads.counts.txt", input),
"-c " + config.param('mapping_quality_filter',
'quality_threshold'))
])
job.name = "mapping_quality_filter." + sample.name
job.samples = [sample]
jobs.append(job)
# Calculate GC bias
# For captured analysis
#if coverage_bed:
#target_input = re.sub(".bam", ".targeted.bam", input)
#job = concat_jobs([
#bedtools.intersect(
#input,
#target_input,
#coverage_bed
#)
#bedtools.coverage(
#target_input,
#re.sub(".bam", ".gc_cov.1M.txt", target_input)
#),
#metrics.gc_bias(
#re.sub(".bam", ".gc_cov.1M.txt", target_input),
#re.sub(".bam", ".GCBias_all.txt", target_input)
#)
#])
# Or for whole genome analysis
#else:
gc_content_file = re.sub(".bam", ".gc_cov.1M.txt", input)
job = bedtools.coverage(input, gc_content_file, coverage_bed)
if coverage_bed:
gc_content_on_target_file = re.sub(".bam",
".gc_cov.1M.on_target.txt",
input)
gc_ontent_target_job = bedtools.intersect(
gc_content_file, gc_content_on_target_file, coverage_bed)
gc_content_file = gc_content_on_target_file
job = concat_jobs([job, gc_ontent_target_job])
job = concat_jobs([
job,
metrics.gc_bias(gc_content_file,
re.sub(".bam", ".GCBias_all.txt", input))
])
job.name = "GC_bias." + sample.name
job.samples = [sample]
jobs.append(job)
job = igvtools.compute_tdf(input, input + ".tdf")
job.name = "igvtools_compute_tdf." + sample.name
job.samples = [sample]
jobs.append(job)
return jobs
def get_metrics_jobs(self, readset):
jobs = []
input_file_prefix = readset.bam + '.'
input = input_file_prefix + "bam"
job = picard.collect_multiple_metrics(
input,
input_file_prefix + "metrics",
reference_sequence=readset.reference_file)
job.name = "picard_collect_multiple_metrics." + readset.name + ".met" + "." + readset.run + "." + readset.lane
jobs.append(job)
if readset.beds:
coverage_bed = readset.beds[0]
full_coverage_bed = (self.output_dir + os.sep + coverage_bed)
else:
coverage_bed = None
full_coverage_bed = None
if coverage_bed:
if (not os.path.exists(full_coverage_bed)) and \
(coverage_bed not in BwaRunProcessingAligner.downloaded_bed_files):
# Download the bed file
command = config.param('DEFAULT',
'fetch_bed_file_command').format(
output_directory=self.output_dir,
filename=coverage_bed)
job = Job([], [full_coverage_bed],
command=command,
name="bed_download." + coverage_bed)
BwaRunProcessingAligner.downloaded_bed_files.append(
coverage_bed)
jobs.append(job)
interval_list = re.sub("\.[^.]+$", ".interval_list", coverage_bed)
if interval_list not in BwaRunProcessingAligner.created_interval_lists:
# Create one job to generate the interval list from the bed file
ref_dict = os.path.splitext(
readset.reference_file)[0] + '.dict'
job = tools.bed2interval_list(ref_dict, full_coverage_bed,
interval_list)
job.name = "interval_list." + coverage_bed
BwaRunProcessingAligner.created_interval_lists.append(
interval_list)
jobs.append(job)
job = picard.calculate_hs_metrics(
input_file_prefix + "bam",
input_file_prefix + "metrics.onTarget.txt",
interval_list,
reference_sequence=readset.reference_file)
job.name = "picard_calculate_hs_metrics." + readset.name + ".hs" + "." + readset.run + "." + readset.lane
jobs.append(job)
jobs.extend(self.verify_bam_id(readset))
job = bvatools.depth_of_coverage(
input,
input_file_prefix + "metrics.targetCoverage.txt",
full_coverage_bed,
other_options=config.param('bvatools_depth_of_coverage',
'other_options',
required=False),
reference_genome=readset.reference_file)
job.name = "bvatools_depth_of_coverage." + readset.name + ".doc" + "." + readset.run + "." + readset.lane
jobs.append(job)
return jobs