def picard_mark_duplicates(self): jobs = [] for sample in self.samples: alignment_file_prefix = os.path.join("alignment", sample.name, sample.name + ".") input = alignment_file_prefix + "realigned.qsorted.bam" output = alignment_file_prefix + "sorted.dup.bam" metrics_file = alignment_file_prefix + "sorted.dup.metrics" job = picard.mark_duplicates([input], output, metrics_file) job.name = "picard_mark_duplicates." + sample.name jobs.append(job) return jobs
def picard_mark_duplicates(self): """ Runs Picard mark duplicates on the sorted bam file. """ jobs = [] for readset in [readset for readset in self.readsets if readset.bam]: input_file_prefix = readset.bam + '.' input = input_file_prefix + "bam" output = input_file_prefix + "dup.bam" metrics_file = readset.bam + ".dup.metrics" job = picard.mark_duplicates([input], output, metrics_file) job.name = "picard_mark_duplicates." + readset.name + ".dup." + self.run_id + "." + str(self.lane_number) jobs.append(job) self.add_copy_job_inputs(jobs) return jobs
def picard_mark_duplicates(self): """ Mark duplicates. Aligned reads per sample are duplicates if they have the same 5' alignment positions (for both mates in the case of paired-end reads). All but the best pair (based on alignment score) will be marked as a duplicate in the BAM file. Marking duplicates is done using [Picard](http://broadinstitute.github.io/picard/). """ jobs = [] for sample in self.samples: alignment_file_prefix = os.path.join("alignment", sample.name, sample.name + ".sorted.") job = picard.mark_duplicates( [alignment_file_prefix + "bam"], alignment_file_prefix + "mdup.bam", alignment_file_prefix + "mdup.metrics" ) job.name = "picard_mark_duplicates." + sample.name jobs.append(job) return jobs
def picard_mark_duplicates(self): """ Mark duplicates. Aligned reads per sample are duplicates if they have the same 5' alignment positions (for both mates in the case of paired-end reads). All but the best pair (based on alignment score) will be marked as a duplicate in the BAM file. Marking duplicates is done using [Picard](http://broadinstitute.github.io/picard/). """ jobs = [] for sample in self.samples: alignment_file_prefix = os.path.join("alignment", sample.name, sample.name + ".sorted.") job = picard.mark_duplicates( [alignment_file_prefix + "bam"], alignment_file_prefix + "mdup.bam", alignment_file_prefix + "mdup.metrics" ) job.name = "picard_mark_duplicates." + sample.name jobs.append(job) return jobs
def picard_mark_duplicates(self): """ Mark duplicates. Aligned reads per sample are duplicates if they have the same 5' alignment positions (for both mates in the case of paired-end reads). All but the best pair (based on alignment score) will be marked as a duplicate in the BAM file. Marking duplicates is done using [Picard](http://broadinstitute.github.io/picard/). """ jobs = [] for sample in self.samples: alignment_file_prefix = os.path.join(self.output_dirs['alignment_output_directory'], sample.name, sample.name + ".") input = alignment_file_prefix + "merged.bam" output = alignment_file_prefix + "sorted.dup.bam" metrics_file = alignment_file_prefix + "sorted.dup.metrics" job = picard.mark_duplicates([input], output, metrics_file) job.name = "picard_mark_duplicates." + sample.name job.sample = [sample] jobs.append(job) report_file = os.path.join(self.output_dirs['report_output_directory'], "ChipSeq.picard_mark_duplicates.md") jobs.append( Job( [os.path.join(self.output_dirs['alignment_output_directory'], sample.name, sample.name + ".sorted.dup.bam") for sample in self.samples], [report_file], command="""\ mkdir -p {report_dir} && \\ cp \\ {report_template_dir}/{basename_report_file} \\ {report_file}""".format( report_template_dir=self.report_template_dir, basename_report_file=os.path.basename(report_file), report_file=report_file, report_dir = self.output_dirs['report_output_directory'] ), report_files=[report_file], name="picard_mark_duplicates_report") ) return jobs
def picard_mark_duplicates(self): """ Mark duplicates. Aligned reads per sample are duplicates if they have the same 5' alignment positions (for both mates in the case of paired-end reads). All but the best pair (based on alignment score) will be marked as a duplicate in the BAM file. Marking duplicates is done using [Picard](http://broadinstitute.github.io/picard/). """ jobs = [] for sample in self.samples: alignment_file_prefix = os.path.join("alignment", sample.name, sample.name + ".") input = alignment_file_prefix + "merged.bam" output = alignment_file_prefix + "sorted.dup.bam" metrics_file = alignment_file_prefix + "sorted.dup.metrics" job = picard.mark_duplicates([input], output, metrics_file) job.name = "picard_mark_duplicates." + sample.name jobs.append(job) report_file = os.path.join("report", "ChipSeq.picard_mark_duplicates.md") jobs.append( Job( [os.path.join("alignment", sample.name, sample.name + ".sorted.dup.bam") for sample in self.samples], [report_file], command="""\ mkdir -p report && \\ cp \\ {report_template_dir}/{basename_report_file} \\ {report_file}""".format( report_template_dir=self.report_template_dir, basename_report_file=os.path.basename(report_file), report_file=report_file ), report_files=[report_file], name="picard_mark_duplicates_report") ) return jobs
def run_cicero(self): """ Fusion detection specializing in internal tandem duplication (ITD) https://genomebiology.biomedcentral.com/articles/10.1186/s13059-020-02043-x https://github.com/stjude/Cicero This software runs as a docker application. However, this can also be installed manually. As of May 2021, versions 0.2.0, 0.3.0 and 1.4.2 are available as modules on the HPF. Also runs RNApeg, a complementary tool to generate the junctions file for use by CICERO. Available on the HPF via RNApeg/20210226 and runs as a singularity container. """ jobs = [] for sample in self.samples: # Get fastq files if len(sample.readsets) > 1: raise Exception("Error: only one read set per sample allowed") if sample.readsets[0].bam: # .bam input fastq_dir = os.path.join("fusions", "picard_sam_to_fastq", sample.name) bam = sample.readsets[0].bam fq1 = os.path.join( self._output_dir, fastq_dir, os.path.basename(re.sub(r"\.bam$", ".pair1.fastq.gz", bam))) fq2 = os.path.join( self._output_dir, fastq_dir, os.path.basename(re.sub(r"\.bam$", ".pair1.fastq.gz", bam))) elif sample.readsets[0].fastq2 and sample.readsets[0].fastq2.split( ".")[-1] == "gz": fq1 = sample.readsets[0].fastq1 fq2 = sample.readsets[0].fastq2 else: raise Exception( "Error: only .bam and .fastq.gz inputs allowed") # Directories tmp_dir = "/localhd/${PBS_JOBID}" # The variable should be unevaluated in the qsub script trim_dir = os.path.join(tmp_dir, "trimmomatic") align_dir = os.path.join(tmp_dir, "star") cicero_dir = os.path.join(tmp_dir, "cicero") rnapeg_dir = os.path.join(tmp_dir, "rnapeg") output_dir = os.path.join("fusions", "cicero", sample.name) # Files fq1_trimmed = os.path.join( trim_dir, "".join([sample.name, ".trimmed.R1.fq.gz"])) fq2_trimmed = os.path.join( trim_dir, "".join([sample.name, ".trimmed.R2.fq.gz"])) fq1_dropped = os.path.join( trim_dir, "".join([sample.name, ".filtered.R1.fq.gz"])) fq2_dropped = os.path.join( trim_dir, "".join([sample.name, ".filtered.R2.fq.gz"])) trim_log = os.path.join(trim_dir, "".join([sample.name, ".trim.log"])) star_bam = os.path.join(align_dir, "Aligned.sortedByCoord.out.bam") dedup_bam = os.path.join(align_dir, "Aligned.sortedByCoord.dedup.bam") dedup_metrics = os.path.join( align_dir, "Aligned.sortedByCoord.dedup.metrics") symlink_bam = os.path.join(cicero_dir, sample.name + ".bam") junction_file = os.path.join( rnapeg_dir, sample.name + ".bam.junctions.tab.shifted.tab") # Jobs trim = trimmomatic.trimmomatic( fq1, fq2, fq1_trimmed, fq1_dropped, fq2_trimmed, fq2_dropped, None, None, config.param("trimmomatic", "adapter_fasta", required=False), trim_log) align = star.align(fq1_trimmed, fq2_trimmed, align_dir, config.param("run_cicero", "genome_build"), rg_id=sample.name, rg_library=sample.name, rg_sample=sample.name, rg_platform="ILLUMINA", sort_bam=True) index = samtools.index(star_bam) # Also indexes for us! idx_file=re.sub(r"\.bam$", ".bai", dedup_bam) dedup = picard.mark_duplicates([star_bam], dedup_bam, dedup_metrics) # RNApeg rna_peg = Job( input_files=[dedup_bam], output_files=[junction_file], module_entries=[("run_cicero", "module_rnapeg")], name="RNApeg", command="""ln -s \\\n{idx_file} \\\n{new_idx_file} && \\ ln -s {bamfile} \\\n{new_bamfile} && \\ singularity exec --cleanenv -B /hpf:/hpf -B /localhd:/localhd -B {outpath}:/results \\ $(which rnapeg.sif) RNApeg.sh -b {new_bamfile} \\\n -f {ref} \\\n -r {reflat}""" .format(bamfile=dedup_bam, ref=config.param("run_cicero", "reference", required=True), reflat=config.param("run_cicero", "reflat", required=True), outpath=rnapeg_dir, idx_file=re.sub(r"\.bam$", ".bai", dedup_bam), new_bamfile=symlink_bam, new_idx_file=symlink_bam + ".bai")) # Cicero cicero = Job( input_files=[dedup_bam, junction_file], output_files=[ os.path.join(cicero_dir, "CICERO_DATADIR", sample.name, "final_fusions.txt") ], module_entries=[("run_cicero", "module_cicero")], name="run_cicero" + sample.name, command= """singularity exec --cleanenv -B /hpf:/hpf -B /localhd:/localhd \\ $CICERO_PATH/CICERO_1.4.2.sif \\ Cicero.sh -n {threads} -b {bamfile} \\\n -g {genome} \\\n -r {reference} \\\n -j {junction} -o {out_dir}""" .format(threads=config.param("run_cicero", "threads", required=True), bamfile=symlink_bam, genome=config.param("run_cicero", "genome", required=True), reference=config.param("run_cicero", "cicero_data", required=True), junction=junction_file, out_dir=cicero_dir)) save_out = Job( input_files=[ os.path.join(cicero_dir, "CICERO_DATADIR", sample.name, "final_fusions.txt") ], output_files=[os.path.join(output_dir, "final_fusions.txt")], name="save_cicero_results" + sample.name, command="""mv {files_to_keep} {target_dir}""".format( files_to_keep=" ".join([ junction_file, os.path.join(cicero_dir, "0*.{err,log}"), # Logs os.path.join(cicero_dir, "CICERO_DATADIR", sample.name, "*.{txt,frame.tab,html}") # # Result files ]), target_dir=output_dir) ) # the files in /localhd/ should be removed automatically upon job end job_mkdir = Job( command="mkdir -p {trim} {align} {cicero} {output} {rnapeg}". format(trim=trim_dir, align=align_dir, cicero=cicero_dir, output=output_dir, rnapeg=rnapeg_dir)) combined_job = concat_jobs([ job_mkdir, trim, align, index, dedup, rna_peg, cicero, save_out ], name="run_cicero." + sample.name) # Replace input and output specification combined_job._output_files = [ os.path.join(output_dir, "final_fusions.txt") ] combined_job.input_files = [fq1, fq2] jobs.append(combined_job) return jobs
def ihec_preprocess_files(self): """ Generate IHEC's files. """ output_dir=self.output_dirs['ihecA_output_directory'] jobs = [] for sample in self.samples: alignment_directory = os.path.join(self.output_dirs['alignment_output_directory'], sample.name) # Find input readset BAMs first from previous bwa_mem_picard_sort_sam job, then from original BAMs in the readset sheet. readset_bams = [os.path.join(alignment_directory, readset.name, readset.name + ".sorted.bam") for readset in sample.readsets] sample_merge_bam = os.path.join(output_dir, sample.name + ".merged.bam") sample_merge_mdup_bam = os.path.join(output_dir, sample.name + ".merged.mdup.bam") sample_merge_mdup_metrics_file = os.path.join(output_dir, sample.name + ".merged.mdup.metrics") mkdir_job = Job(command="mkdir -p " + output_dir) # If this sample has one readset only, create a sample BAM symlink to the readset BAM, along with its index. if len(sample.readsets) == 1: readset_bam = readset_bams[0] if os.path.isabs(readset_bam): target_readset_bam = readset_bam else: target_readset_bam = os.path.relpath(readset_bam, output_dir) job = concat_jobs([ mkdir_job, Job([readset_bam], [sample_merge_bam], command="ln -s -f " + target_readset_bam + " " + sample_merge_bam, removable_files=[sample_merge_bam]), ], name="ihecs_preprocess_symlink." + sample.name) elif len(sample.readsets) > 1: job = concat_jobs([ mkdir_job, picard.merge_sam_files(readset_bams, sample_merge_bam) ]) job.name = "ihecs_preprocess_merge." + sample.name jobs.append(job) tmp_dir = config.param('ihec_preprocess_files', 'tmp_dir') job = concat_jobs([Job(command = "export TMPDIR={tmp_dir}".format(tmp_dir = tmp_dir)), picard.mark_duplicates([sample_merge_bam], sample_merge_mdup_bam, sample_merge_mdup_metrics_file)]) job.name = "ihecs_preprocess_mark_duplicates." + sample.name jobs.append(job) return jobs