def run_pipeline(self, pipeline_args, pipeline_config):
# create variables from parser if wanted
bamFiles = pipeline_args['bam:lib']
outputDir = pipeline_args['output']
adapter = pipeline_args['adapter']
numThreads = pipeline_args['threads']
# Create output directory
subprocess.call(['mkdir', outputDir])
# Software
cutadapt = Software('cutadapt', pipeline_config['cutadapt']['path'])
star = Software('STAR', pipeline_config['STAR']['path'])
bedtools = Software('bedtools', pipeline_config['bedtools']['path'])
bowtie2 = Software('bowtie2', pipeline_config['bowtie2']['path'])
samtools = Software('samtools', pipeline_config['samtools']['path'])
samtools_sort = Software('samtools sort',
pipeline_config['samtools']['path'])
read_distribution = Software(
'read_distribution.py',
pipeline_config['read_distribution']['path'])
featureCounts = Software('featureCounts',
pipeline_config['featureCounts']['path'])
fastQC = Software('FastQC', pipeline_config['FastQC']['path'])
picard = Software('picard', pipeline_config['picard']['path'])
# Change these to just be done in python script?
# Common software tools
awk = Software('awk', 'awk')
sort = Software('sort', 'sort')
uniq = Software('uniq', 'uniq')
paste = Software('paste', 'paste')
cat = Software('cat', 'cat')
grep = Software('grep', 'grep')
# Directories and Files
pathToGenomeDir = pipeline_config['STAR']['genomeDir']
pathToGenome = pipeline_config['bowtie2']['genome_ref']
pathToGtf = pipeline_config['STAR']['GTF_ref']
pathTo_hg19_bed = pipeline_config['read_distribution']['hg19_bed']
pathTo_hg19_bed_start100 = pipeline_config['bedtools']['hg19_start100']
pathTo_grch37_bed = pipeline_config['bedtools']['grch37_bed']
pathTo_genomeFasta = pipeline_config['picard']['genomeFasta']
'''
remove adaptor and trim
adaptor sequence: AGATCGGAAGAGCACACGTCT
-m 25 discard any reads shorter than 25 nucleotides
keep only reads that had the adaptor sequence --discard-untrimmed
cutadapt -a AGATCGGAAGAGCACACGTCT -m 25 --discard-untrimmed {filename}.fastq.gz
> {filename}_trimmed.fastq.gz 2> {filename}_report.txt
Remove adapters
Only keep reads with adapters, otherwise artifact
Discard reads shorter than 25 bp
'''
# Keep track of Bids in pipeline
bid_list = []
for bamLib in bamFiles:
bid_list.append(bamLib.split(':')[-1])
'''
Sort and extract uniquely mapped reads for QC and further analyses
samtools view -H $file > header.sam
samtools view $file | grep -w NH:i:1 | cat header.sam - | samtools view -bS - | samtools sort - ${filename}_uniq_sorted
rm header.sam
Using this file for the rest of the analysis
'''
for bamLib in bamFiles:
bam, bid = bamLib.split(':')
newDir = new_dir(outputDir, bid)
samtools.run(
Parameter('view'),
Parameter('-H'),
Parameter(bam), # star outfile name
Redirect(stream=Redirect.STDOUT,
dest=os.path.join(newDir,
'{}.header.sam'.format(bid))))
samtools.run(
Parameter('view'),
Parameter(bam), # star outfile name
Pipe(
grep.pipe(
Parameter('-w'), Parameter('NH:i:1'),
Pipe(
cat.pipe(
Parameter(
os.path.join(newDir,
'{}.header.sam'.format(bid)),
'-'),
Pipe(
samtools.pipe(
Parameter('view'),
Parameter('-bS', '-'),
Pipe(
samtools.pipe(
Parameter('sort'),
Parameter(
'-', '-o',
'{}/{}.uniq_sorted.bam'.
format(newDir,
bid)))))))))))
# subprocess.call(['rm', '{}/{}.header.sam'.format(newDir, bid)])
'''
SeQC to evaluate percent reads mapped to each genomic features
read_distribution.py -r hg19_RefSeq.bed12 -i $file
'''
for bid in bid_list:
newDir = new_dir(outputDir, bid)
read_distribution.run(
Parameter('-r'),
Parameter(pathTo_hg19_bed),
Parameter('-i'),
Parameter('{}/{}.uniq_sorted.bam'.format(newDir, bid)),
Redirect(stream=Redirect.STDOUT,
dest=os.path.join(
newDir, '{}.read_distribution.log'.format(bid))),
shell=True)
'''
codon periodicity
annotation=/glusterfs/users/ashieh/annotations/hg19_ccds_exons_plus_start100.bed
bedtools intersect -a {annotation} -b {uniq.bam} -s -bed -wa -wb > intersect_start100
awk -v OFS='\t' '{print ($2-($14+100))}' ${filename}_intersect_start100.bed
| sort | uniq -c > ${filename}_relative_pos_aggregate.table
'''
# bedtools intersect -a {annotation} -b {uniq.bam} -s -bed -wa -wb > intersect_start100
for bid in bid_list:
newDir = new_dir(outputDir, bid)
bedtools.run(
Parameter('intersect'),
Parameter('-a {}'.format(pathTo_hg19_bed_start100)),
Parameter('-b {}/{}.uniq_sorted.bam'.format(newDir, bid)),
Parameter('-s'),
Parameter('-bed'),
Parameter('-wa'),
Parameter('-wb'),
Redirect(stream=Redirect.STDOUT,
dest=os.path.join(
newDir, '{}.intersect_start100.bed'.format(bid))),
shell=True)
awk.run(
Parameter('-v'), Parameter("OFS='\\t'"),
Parameter('{print ($8-($2+100))}'),
Parameter('{}/{}.intersect_start100.bed'.format(newDir, bid)),
Pipe(
sort.pipe(
Pipe(
uniq.pipe(
Parameter('-c'),
Redirect(stream=Redirect.STDOUT,
dest=os.path.join(
newDir,
'{}_relative_pos_aggregate.table'.
format(bid))))))))
for bid in bid_list:
newDir = new_dir(outputDir, bid)
rpaFile = open(
'{dir}/{bid}_relative_pos_aggregate.table'.format(dir=newDir,
bid=bid),
'rb')
myDict = {}
for i in range(-30, 31):
myDict[i] = 0
for line in rpaFile:
Frequency, start = line.strip().split(' ')
if int(start) >= -30 and int(start) <= 30:
print start
myDict[int(start)] = Frequency
# print times
freqs = []
starts = []
for i in range(-30, 31):
starts.append(i)
freqs.append(myDict[i])
# print freqs
fig, ax = plt.subplots()
# plt.set_title('{} codon periodicity'.format(bid))
plt.xlabel("-30 to 30 relative position")
plt.ylabel("Frequency")
plt.bar(starts, freqs)
fig.savefig('{dir}/{bid}_codon_periodicity_plot.png'.format(
dir=newDir, bid=bid))
'''
Picard tools
java -jar picard.jar CollectMultipleMetrics
I=2017-221.uniq_sorted.bam
O= multiple_metrics
R=GRCh37.p13.genome.fa
java -jar picard.jar CollectGcBiasMetrics
I= .uniq
O=gc_bias_metrics.txt
CHART=gc_bias_metrics.pdf
S=summary_metrics.txt
R=reference_sequence.fasta
java -jar picard.jar CollectRnaSeqMetrics
I=input.bam
O=output.RNA_Metrics
REF_FLAT=ref_flat.txt
STRAND=FIRST_READ_TRANSCRIPTION_STRAND
java -jar picard.jar MarkDuplicates
I=input.bam
O=marked_duplicates.bam
M=marked_dup_metrics.txt
ASSUME_SORTED=true
'''
for bid in bid_list:
newDir = new_dir(outputDir, bid)
picard.run(
Parameter('CollectMultipleMetrics'),
Parameter('I={}/{}.uniq_sorted.bam'.format(newDir,
bid)), # input
Parameter('O={}/{}.multiple_metrics'.format(newDir,
bid)), # output
Parameter('R={}'.format(pathTo_genomeFasta)) # genomeReference
)
picard.run(
Parameter('CollectGcBiasMetrics'),
Parameter('I={}/{}.uniq_sorted.bam'.format(newDir,
bid)), # input
Parameter('O={}/{}.gc_bias_metrics'.format(newDir,
bid)), # output
Parameter('CHART={}/{}.gc_bias_metrics.pdf'.format(
newDir, bid)), # chart
Parameter('S={}/{}.summary_metrics'.format(
newDir, bid)), # summary metrics
Parameter(
'R={}'.format(pathTo_genomeFasta)) # genome reference
)
picard.run(
Parameter('CollectRnaSeqMetrics'),
Parameter('I={}/{}.uniq_sorted.bam'.format(newDir,
bid)), # input
Parameter('O={}/{}.RNA_Metrics'.format(newDir, bid)), # output
Parameter('REF_FLAT={}/{}'.format(newDir, bid)), # ref_flat
Parameter(
'STRAND=FIRST_READ_TRANSCRIPTION_STRAND') # strandedness
)
picard.run(
Parameter('MarkDuplicates'),
Parameter('I={}/{}.uniq_sorted.bam'.format(newDir,
bid)), # input
Parameter('O={}/{}.marked_duplicates.bam'.format(
newDir, bid)), # output
Parameter(
'M={}/{}.marked_dup_metrics.txt'), # marked dup metrics
Parameter('ASSUME_SORTED=true') # sorted
)
'''
subread: featureCounts
featureCounts -a /path_to_gtf/gencode.v19.annotation.gtf -o <bid>.featureCounts <bid>.uniq_sorted.bam
'''
for bid in bid_list:
newDir = new_dir(outputDir, bid)
featureCounts.run(
Parameter('-a', '{}'.format(pathToGtf)), # gtf
Parameter('-s', '1'), # strand-specific read counting
Parameter('-o', '{}/{}.featureCounts'.format(newDir,
bid)), # output
Parameter('{}/{}.uniq_sorted.bam'.format(newDir, bid)) # input
)
def run_pipeline(self, pipeline_args, pipeline_config):
# Instantiate variables from argparse
read_pairs = pipeline_args['reads']
output_dir = os.path.abspath(pipeline_args['output'])
logs_dir = os.path.join(output_dir, 'logs')
lib_prefix = pipeline_args['lib']
step = int(pipeline_args['step'])
forward_adapter = pipeline_args['forward_adapter']
reverse_adapter = pipeline_args['reverse_adapter']
# Create output, tmp, and logs directories
tmp_dir = os.path.join(output_dir, 'tmp')
subprocess.call(['mkdir', '-p', output_dir, tmp_dir, logs_dir])
# Keep list of items to delete
staging_delete = [tmp_dir]
bwa_bam_outs = []
qc_data = {
'total_raw_reads_counts': [],
'trimmed_reads_counts': [],
# TODO Find a better way to store FastQC results
'num_reads_mapped': [],
'percent_duplicate_reads': '0',
'num_unique_reads_mapped': [], # TODO This isn't implemented
'num_mtDNA_reads_mapped': [], # TODO This isn't implemented
'num_reads_mapped_after_filtering': '-1', # TODO This isn't implemented
'num_peaks_called': '-1',
# TODO Get number of peaks in annotation sites
}
# Instantiate software instances
cutadapt = Software('cutadapt', pipeline_config['cutadapt']['path'])
fastqc = Software('FastQC', pipeline_config['fastqc']['path'])
bwa_aln = Software('BWA aln', pipeline_config['bwa']['path'] + ' aln')
bwa_sampe = Software('BWA sampe', pipeline_config['bwa']['path'] + ' sampe')
samtools_view = Software('samtools view',
pipeline_config['samtools']['path'] + ' view')
samtools_flagstat = Software('samtools flagstat',
pipeline_config['samtools']['path'] + ' flagstat')
samtools_index = Software('samtools index',
pipeline_config['samtools']['path'] + ' index')
novosort = Software('novosort', pipeline_config['novosort']['path'])
picard_mark_dup = Software('Picard MarkDuplicates',
pipeline_config['picard']['path'] + ' MarkDuplicates')
picard_insert_metrics = Software('Picard CollectInsertSizeMetrics',
pipeline_config['picard']['path'] + ' CollectInsertSizeMetrics')
bedtools_bamtobed = Software('bedtools bamtobed',
pipeline_config['bedtools']['path'] + ' bamtobed')
bedtools_sort = Software('bedtools sort', pipeline_config['bedtools']['path'] + ' sort')
bedtools_merge = Software('bedtools merge', pipeline_config['bedtools']['path'] + ' merge')
bedtools_intersect = Software('bedtools intersect',
pipeline_config['bedtools']['path'] + ' intersect')
homer_maketagdir = Software('HOMER makeTagDirectory',
pipeline_config['makeTagDirectory']['path'])
homer_findpeaks = Software('HOMER findPeaks', pipeline_config['findPeaks']['path'])
homer_pos2bed = Software('HOMER pos2bed', pipeline_config['pos2bed']['path'])
if step <= 1:
for i, read_pair in enumerate(read_pairs):
read1, read2 = read_pair.split(':')
# QC: Get raw fastq read counts
qc_data['total_raw_reads_counts'].append([
str(int(self.count_gzipped_lines(read1))/4),
str(int(self.count_gzipped_lines(read2))/4)
])
trimmed_read1_filename = os.path.join(output_dir,
lib_prefix + '_{}_read1.trimmed.fastq.gz'.format(i))
trimmed_read2_filename = os.path.join(output_dir,
lib_prefix + '_{}_read2.trimmed.fastq.gz'.format(i))
cutadapt.run(
Parameter('--quality-base=33'),
Parameter('--minimum-length=5'),
Parameter('-q', '30'), # Minimum quality score
Parameter('--output={}'.format(trimmed_read1_filename)),
Parameter('--paired-output={}'.format(trimmed_read2_filename)),
Parameter('-a', forward_adapter if forward_adapter else 'ZZZ'),
Parameter('-A', reverse_adapter if reverse_adapter else 'ZZZ'),
Parameter(read1),
Parameter(read2),
Redirect(stream=Redirect.STDOUT, dest=os.path.join(logs_dir, 'cutadapt.summary.log'))
)
# QC: Get trimmed fastq read counts
qc_data['trimmed_reads_counts'].append([
str(int(self.count_gzipped_lines(trimmed_read1_filename))/4),
str(int(self.count_gzipped_lines(trimmed_read2_filename))/4)
])
staging_delete.extend([trimmed_read1_filename, trimmed_read2_filename])
read_pairs[i] = ':'.join([trimmed_read1_filename, trimmed_read2_filename])
if step <= 2:
# Make FastQC directory
fastqc_output_dir = os.path.join(output_dir, 'fastqc')
subprocess.call(['mkdir', '-p', fastqc_output_dir])
for i, read_pair in enumerate(read_pairs):
for read in read_pair.split(':'):
fastqc.run(
Parameter('--outdir={}'.format(fastqc_output_dir)),
Parameter(read)
)
bwa_aln.run(
Parameter('-t', pipeline_config['bwa']['threads']),
Parameter(pipeline_config['bwa']['index-dir']),
Parameter(read),
Redirect(stream=Redirect.STDOUT, dest='{}.sai'.format(read))
)
staging_delete.append('{}.sai'.format(read))
if step <= 3:
for i, read_pair in enumerate(read_pairs):
read1, read2 = read_pair.split(':')
bwa_bam_output = os.path.join(output_dir, '{}.{}.bam'.format(lib_prefix, i))
bwa_sampe.run(
Parameter('-a', '2000'), # Maximum insert size
Parameter('-n', '1'),
Parameter(pipeline_config['bwa']['index-dir']),
Parameter('{}.sai'.format(read1)),
Parameter('{}.sai'.format(read2)),
Parameter(read1),
Parameter(read2),
Redirect(stream=Redirect.STDERR, dest=os.path.join(logs_dir, 'bwa_sampe.log')),
Pipe(
samtools_view.pipe(
Parameter('-hSb'),
Parameter('-o', bwa_bam_output),
Parameter('-') # Get input from stdin
)
)
)
bwa_bam_outs.append(bwa_bam_output)
if step <= 4:
for i, bwa_bam in enumerate(bwa_bam_outs):
samtools_flagstat.run(
Parameter(bwa_bam),
Redirect(stream=Redirect.STDOUT, dest=bwa_bam + '.flagstat')
)
# QC: Get number of mapped reads from this BAM
try:
with open(bwa_bam + '.flagstat') as flagstats:
flagstats_contents = flagstats.read()
target_line = re.search(r'(\d+) \+ \d+ mapped', flagstats_contents)
if target_line is not None:
qc_data['num_reads_mapped'].append(str(int(target_line.group(1))/2))
else:
qc_data['num_reads_mapped'].append('0')
except:
qc_data['num_reads_mapped'].append('Could not open flagstats {}'.format(
bwa_bam + '.flagstat'
))
sortmerged_bam = os.path.join(output_dir, '{}.sortmerged.bam'.format(lib_prefix))
steric_filter_bam = os.path.join(output_dir, '{}.steric.bam'.format(lib_prefix))
duprm_bam = os.path.join(output_dir, '{}.duprm.bam'.format(lib_prefix))
unique_bam = os.path.join(output_dir, '{}.unique.bam'.format(lib_prefix))
unmappedrm_bam = os.path.join(output_dir, '{}.unmappedrm.bam'.format(lib_prefix))
chrmrm_bam = os.path.join(output_dir, '{}.chrmrm.bam'.format(lib_prefix))
novosort.run(
Parameter('--threads', pipeline_config['novosort']['threads']),
Parameter('--tmpcompression', '6'),
Parameter('--tmpdir', tmp_dir),
Parameter(*[bam for bam in bwa_bam_outs]),
Redirect(stream=Redirect.STDOUT, dest=sortmerged_bam),
Redirect(stream=Redirect.STDERR, dest=os.path.join(logs_dir, 'novosort.log'))
)
# This creates a dependency on PySam
# Removes reads with template length < 38 due to steric hindrence
samtools_index.run(Parameter(sortmerged_bam))
sortmerged_bam_alignmentfile = pysam.AlignmentFile(sortmerged_bam, 'rb')
steric_filter_bam_alignmentfile = pysam.AlignmentFile(steric_filter_bam, 'wb',
template=sortmerged_bam_alignmentfile)
for read in sortmerged_bam_alignmentfile.fetch():
if abs(int(read.template_length)) >= STERIC_HINDRANCE_CUTOFF:
steric_filter_bam_alignmentfile.write(read)
sortmerged_bam_alignmentfile.close()
steric_filter_bam_alignmentfile.close()
# Mark and remove duplicates
markduplicates_metrics_filepath = os.path.join(logs_dir,
'mark_dup.metrics')
picard_mark_dup.run(
Parameter('INPUT={}'.format(steric_filter_bam)),
Parameter('OUTPUT={}'.format(duprm_bam)),
Parameter('TMP_DIR={}'.format(tmp_dir)),
Parameter('METRICS_FILE={}'.format(markduplicates_metrics_filepath)),
Parameter('REMOVE_DUPLICATES=true'),
Parameter('VALIDATION_STRINGENCY=LENIENT'),
Redirect(stream=Redirect.BOTH, dest=os.path.join(logs_dir, 'mark_dup.log'))
)
# QC: Get percent duplicates
try:
with open(markduplicates_metrics_filepath) as markdup_metrics:
for line in markdup_metrics:
if line[FIRST_CHAR] == '#':
continue
record = line.strip().split('\t')
if len(record) == 9:
if re.match(r'\d+', record[7]) is not None:
qc_data['percent_duplicate_reads'] = record[7]
except:
qc_data['percent_duplicate_reads'] = 'Could not open MarkDuplicates metrics'
# Filter down to uniquely mapped reads
samtools_view.run(
Parameter('-b'),
Parameter('-F', '256'),
Parameter('-q', '10'),
Parameter('-o', unique_bam),
Parameter(duprm_bam)
)
# Remove unmapped reads
samtools_view.run(
Parameter('-b'),
Parameter('-F', '12'),
Parameter('-o', unmappedrm_bam),
Parameter(unique_bam)
)
# Create BAM index, then remove chrM
samtools_index.run(
Parameter(unmappedrm_bam)
)
# Remove chrM
all_chr = [Parameter('chr{}'.format(chromosome)) for chromosome in map(str, range(1, 23)) + ['X', 'Y']]
samtools_view.run(
Parameter('-b'),
Parameter('-o', chrmrm_bam),
Parameter(unmappedrm_bam),
*all_chr
)
# Stage delete for temporary files
staging_delete.extend([
sortmerged_bam,
sortmerged_bam + '.bai', # BAM index file
steric_filter_bam,
unique_bam,
duprm_bam,
unmappedrm_bam,
unmappedrm_bam + '.bai', # BAM index file
chrmrm_bam
])
if step <= 5:
# Generate filename for final processed BAM and BED
processed_bam = os.path.join(output_dir, '{}.processed.bam'.format(lib_prefix))
unshifted_bed = os.path.join(output_dir, '{}.unshifted.bed'.format(lib_prefix))
processed_bed = os.path.join(output_dir, '{}.processed.bed'.format(lib_prefix))
# staging_delete.append(unshifted_bed)
# Generate filename for chrM removed BAM
chrmrm_bam = os.path.join(output_dir, '{}.chrmrm.bam'.format(lib_prefix))
# Remove blacklisted genomic regions
bedtools_intersect.run(
Parameter('-v'),
Parameter('-abam', chrmrm_bam),
Parameter('-b', pipeline_config['bedtools']['blacklist-bed']),
Parameter('-f', '0.5'),
Redirect(stream=Redirect.STDOUT, dest=processed_bam)
)
# QC: Generate insert size metrics PDF
picard_insert_metrics.run(
Parameter('INPUT={}'.format(processed_bam)),
Parameter('OUTPUT={}'.format(os.path.join(logs_dir, lib_prefix + '.insertsize.metrics'))),
Parameter('HISTOGRAM_FILE={}'.format(os.path.join(logs_dir, lib_prefix + '.insertsize.pdf')))
)
# Generate index for processed BAM
samtools_index.run(
Parameter(processed_bam)
)
# Convert BAM to BED
bedtools_bamtobed.run(
Parameter('-i', processed_bam),
Redirect(stream=Redirect.STDOUT, dest=unshifted_bed)
)
staging_delete.append(unshifted_bed)
# Shifting + strand by 4 and - strand by -5, according to
# the ATACseq paper
# This used to be bedtools shift, but they are fired
self.shift_reads(
input_bed_filepath=unshifted_bed,
output_bed_filepath=processed_bed,
log_filepath=os.path.join(logs_dir, 'shift_reads.logs'),
genome_sizes_filepath=pipeline_config['bedtools']['genome-sizes'],
minus_strand_shift=MINUS_STRAND_SHIFT,
plus_strand_shift=PLUS_STRAND_SHIFT
)
if step <= 6:
processed_bed = os.path.join(output_dir, '{}.processed.bed'.format(lib_prefix))
homer_tagdir = os.path.join(output_dir, '{}_tagdir'.format(lib_prefix))
unsorted_peaks = os.path.join(output_dir, '{}.unsorted.peaks.bed'.format(lib_prefix))
sorted_peaks = os.path.join(output_dir, '{}.sorted.peaks.bed'.format(lib_prefix))
merged_peaks = os.path.join(output_dir, '{}.peaks.bed'.format(lib_prefix))
# Populate HOMER tag directory
homer_maketagdir.run(
Parameter(homer_tagdir),
Parameter('-format', 'bed'),
Parameter(processed_bed),
Redirect(stream=Redirect.BOTH, dest=os.path.join(logs_dir, 'maketagdir.log'))
)
# Run HOMER peak calling program
homer_findpeaks.run(
Parameter(homer_tagdir),
Parameter('-fragLength', '0'),
Parameter('-fdr', '0.01'),
Parameter('-localSize', '50000'),
Parameter('-o', 'auto'),
Parameter('-style', 'dnase'),
Parameter('-size', '150'),
Parameter('-minDist', '50'),
Redirect(stream=Redirect.BOTH, dest=os.path.join(logs_dir, 'findpeaks.log'))
)
# Convert HOMER peaks file to bed format
homer_pos2bed.run(
Parameter(os.path.join(homer_tagdir, 'peaks.txt')),
Redirect(stream=Redirect.STDOUT, dest=unsorted_peaks),
Redirect(stream=Redirect.STDERR, dest=os.path.join(logs_dir, 'pos2bed.log'))
)
# Sort called peaks bed file
bedtools_sort.run(
Parameter('-i', unsorted_peaks),
Redirect(stream=Redirect.STDOUT, dest=sorted_peaks)
)
# Merge peaks to create final peaks file
bedtools_merge.run(
Parameter('-i', sorted_peaks),
Redirect(stream=Redirect.STDOUT, dest=merged_peaks)
)
# Stage delete for temporary files
staging_delete.extend([
unsorted_peaks,
sorted_peaks
])
# QC: Output QC data to file
with open(os.path.join(logs_dir, 'qc_metrics.txt'), 'w') as qc_data_file:
qc_data_file.write(str(qc_data) + '\n')
# Delete temporary files
for delete_file in staging_delete:
subprocess.call(['rm', '-rf', delete_file])
def add_pipeline_args(self, parser):
parser.add_argument(
'--fastq:lib',
required=True,
nargs='*',
help='Fastq input for pipeline:library name(prefix for files)')
parser.add_argument('--output',
required=True,
help='Where pipeline output should go')
parser.add_argument('--adapter',
default='AGATCGGAAGAGCACACGTCT',
help='Adapter sequence for trimming')
parser.add_argument(
'--threads',
default=defaultThreads,
help='Threads to be used for multi-threaded programs. Default is 8'
)
# chunky run RiboSeq_pipe.py --fastqs
# /mnt/cinder/thomas/RiboSeq/Lane5/AWS-3_S3_L005_R1_001.fastq.gz
# --output /mnt/cinder/thomas/RiboSeq/test --threads
# create variables from parser if wanted
fastqFiles = pipeline_args['fastq:lib']
outputDir = pipeline_args['output']
adapter = pipeline_args['adapter']
numThreads = pipeline_args['threads']
# Create output directory
subprocess.call(['mkdir', outputDir])
# Software
cutadapt = Software('cutadapt', pipeline_config['cutadapt']['path'])
star = Software('STAR', pipeline_config['STAR']['path'])
bedtools = Software('bedtools', pipeline_config['bedtools']['path'])
bowtie2 = Software('bowtie2', pipeline_config['bowtie2']['path'])
samtools = Software('samtools', pipeline_config['samtools']['path'])
samtools_sort = Software('samtools sort',
pipeline_config['samtools']['path'])
read_distribution = Software(
'read_distribution.py',
pipeline_config['read_distribution']['path'])
featureCounts = Software('featureCounts',
pipeline_config['featureCounts']['path'])
fastQC = Software('FastQC', pipeline_config['FastQC']['path'])
picard = Software('picard', pipeline_config['picard']['path'])
# Change these to just be done in python script?
# Common software tools
awk = Software('awk', 'awk')
sort = Software('sort', 'sort')
uniq = Software('uniq', 'uniq')
paste = Software('paste', 'paste')
cat = Software('cat', 'cat')
grep = Software('grep', 'grep')
# Directories and Files
pathToGenomeDir = pipeline_config['STAR']['genomeDir']
pathToGenome = pipeline_config['bowtie2']['genome_ref']
pathToGtf = pipeline_config['STAR']['GTF_ref']
pathTo_hg19_bed = pipeline_config['read_distribution']['hg19_bed']
pathTo_hg19_bed_start100 = pipeline_config['bedtools']['hg19_start100']
pathTo_grch37_bed = pipeline_config['bedtools']['grch37_bed']
pathTo_genomeFasta = pipeline_config['picard']['genomeFasta']
pathTo_ref_flat = pipeline_config['picard']['refFlat']
'''
remove adaptor and trim
adaptor sequence: AGATCGGAAGAGCACACGTCT
-m 25 discard any reads shorter than 25 nucleotides
keep only reads that had the adaptor sequence --discard-untrimmed
cutadapt -a AGATCGGAAGAGCACACGTCT -m 25 --discard-untrimmed {filename}.fastq.gz
> {filename}_trimmed.fastq.gz 2> {filename}_report.txt
Remove adapters
Only keep reads with adapters, otherwise artifact
Discard reads shorter than 25 bp
'''
# Keep track of Bids in pipeline
bid_list = []
for fastqlib in fastqFiles:
bid_list.append(fastqlib.split(':')[-1])
# Cutadapt
for fastqlib in fastqFiles:
fastq, bid = fastqlib.split(':')
newDir = new_dir(outputDir, bid)
# Make new directories to store data
subprocess.call(['mkdir', newDir])
# consider multi-threading by splitting in multiple files and then combining
cutadapt.run(
Parameter('--quality-base=33'),
Parameter('--minimum-length=25'),
Parameter('--discard-untrimmed'),
Parameter('--output={}/{}.trimmed.fastq.gz'.format(
newDir, bid)),
# Parameter('-a', forward_adapter if forward_adapter else 'AGATCGGAAGAGCACACGTCT'),
Parameter('-a', adapter),
Parameter(fastq),
Redirect(stream=Redirect.STDOUT,
dest=os.path.join(
newDir, '{}.cutadapt.summary.log'.format(bid))))
'''
Bowtie2
bowtie2 --seedlen=23 --un-fq=${filename}_filtered.fq -x $genome -U $file
-S | samtools view -Sb - > ${filename}.rts.bam
Remove snoRNA, rRNA, tRNA, keep only mRna for alignment
'''
for bid in bid_list:
newDir = new_dir(outputDir, bid)
bowtie2.run(
Parameter('--seedlen=23'),
Parameter('--threads', numThreads),
Parameter('--un-gz {}/{}_filtered.fq.gz'.format(newDir, bid)),
Parameter('-x', pathToGenome), # Path to rtsRNA_seqs files
Parameter('-U', '{}/{}.trimmed.fastq.gz'.format(newDir, bid)),
Parameter('-S'),
Parameter('{}/{}.rts.sam'.format(newDir, bid)),
Redirect(stream=Redirect.STDOUT,
dest=os.path.join(newDir,
'{}.bowtie2.log'.format(bid))),
Redirect(stream=Redirect.STDERR,
dest=os.path.join(newDir,
'{}.bowtie2.log2'.format(bid))),
shell=True # Look into changing
)
# This doesn't work
samtools.run(
Parameter('view'),
Parameter('-Sb'),
Parameter('{}/{}.rts.sam'.format(newDir, bid)),
Redirect(stream=Redirect.STDOUT,
dest=os.path.join(newDir, '{}.rts.bam'.format(bid))),
)
'''
Star
STAR --runThreadN 6 --sjdbGTFfile gtfFile --outSAMtype BAM Unsorted
--outFileNamePrefix {filename}_ --genomeDir /path/to/genome/index
--genomeFastaFiles --readFilesIn
{filename}_filtered.fq.gz --readFilesCommand zcat
Basically RNAseq at this point
Align the kept reads from bowtie to the genome
'''
# Only load the genome one time: genomeLoad = 'LoadAndKeep'.....Doesn't really work
for bid in bid_list:
newDir = new_dir(outputDir, bid)
# remove genome from memory on last run
# genomeLoad = 'LoadAndRemove'
star.run(
Parameter(
'--runThreadN',
numThreads), # Change to command line parameter --threads
Parameter('--sjdbGTFfile', pathToGtf),
Parameter('--outSAMtype', 'BAM', 'Unsorted'),
Parameter('--outFileNamePrefix', '{}/{}_'.format(newDir, bid)),
Parameter('--genomeDir', pathToGenomeDir),
# Parameter('--genomeLoad', genomeLoad), broken
Parameter('--readFilesIn',
'{}/{}_filtered.fq.gz'.format(newDir, bid)),
Parameter('--readFilesCommand zcat') # reads gzipped files
)
'''
Sort and extract uniquely mapped reads for QC and further analyses
samtools view -H $file > header.sam
samtools view $file | grep -w NH:i:1 | cat header.sam - | samtools view -bS - | samtools sort - ${filename}_uniq_sorted
rm header.sam
Using this file for the rest of the analysis
'''
for bid in bid_list:
newDir = new_dir(outputDir, bid)
samtools.run(
Parameter('view'),
Parameter('-H'),
Parameter('{}/{}_Aligned.out.bam'.format(
newDir, bid)), # star outfile name
Redirect(stream=Redirect.STDOUT,
dest=os.path.join(newDir,
'{}.header.sam'.format(bid))))
samtools.run(
Parameter('view'),
Parameter('{}/{}_Aligned.out.bam'.format(
newDir, bid)), # star outfile name
Pipe(
grep.pipe(
Parameter('-w'), Parameter('NH:i:1'),
Pipe(
cat.pipe(
Parameter(
os.path.join(newDir,
'{}.header.sam'.format(bid)),
'-'),
Pipe(
samtools.pipe(
Parameter('view'),
Parameter('-bS', '-'),
Pipe(
samtools.pipe(
Parameter('sort'),
Parameter(
'-', '-o',
'{}/{}.uniq_sorted.bam'.
format(newDir,
bid)))))))))))
# subprocess.call(['rm', '{}/{}.header.sam'.format(newDir, bid)])
'''
rSeQC to evaluate percent reads mapped to each genomic features
read_distribution.py -r hg19_RefSeq.bed12 -i $file
'''
for bid in bid_list:
newDir = new_dir(outputDir, bid)
read_distribution.run(
Parameter('-r'),
Parameter(pathTo_hg19_bed),
Parameter('-i'),
Parameter('{}/{}.uniq_sorted.bam'.format(newDir, bid)),
Redirect(stream=Redirect.STDOUT,
dest=os.path.join(
newDir, '{}.read_distribution.log'.format(bid))),
shell=True)
'''
codon periodicity
annotation=/glusterfs/users/ashieh/annotations/hg19_ccds_exons_plus_start100.bed
bedtools intersect -a {annotation} -b {uniq.bam} -s -bed -wa -wb > intersect_start100
awk -v OFS='\t' '{print ($2-($14+100))}' ${filename}_intersect_start100.bed
| sort | uniq -c > ${filename}_relative_pos_aggregate.table
'''
# bedtools intersect -a {annotation} -b {uniq.bam} -s -bed -wa -wb > intersect_start100
for bid in bid_list:
newDir = new_dir(outputDir, bid)
bedtools.run(
Parameter('intersect'),
Parameter('-a {}'.format(pathTo_hg19_bed_start100)),
Parameter('-b {}/{}.uniq_sorted.bam'.format(newDir, bid)),
Parameter('-s'),
Parameter('-bed'),
Parameter('-wa'),
Parameter('-wb'),
Redirect(stream=Redirect.STDOUT,
dest=os.path.join(
newDir, '{}.intersect_start100.bed'.format(bid))),
shell=True)
start100_file = open(
'{}/{}.intersect_start100.bed'.format(newDir, bid), 'rb')
relativePos_file = open(
'{}/{}_relative_pos_aggregate.table'.format(newDir, bid), 'wb')
distanceList = []
for line in start100_file:
splitLine = line.split('\t')
# Really is relative start
if len(splitLine) >= 7:
distance = int(splitLine[7]) - (int(splitLine[1]) + 100)
distanceList.append(distance)
distanceList.sort()
distanceCounting = Counter(distanceList)
for key, value in distanceCounting.iteritems():
relativePos_file.write("{}\t{}\n".format(value, key))
# Create chart of relative_positions_aggregate to see codon periodicity
for bid in bid_list:
newDir = new_dir(outputDir, bid)
rpaFile = open(
'{dir}/{bid}_relative_pos_aggregate.table'.format(dir=newDir,
bid=bid),
'rb')
myDict = {}
for i in range(-30, 31):
myDict[i] = 0
for line in rpaFile:
Frequency, start = line.strip().split(' ')
if int(start) >= -30 and int(start) <= 30:
# print start
myDict[int(start)] = Frequency
# Change to log scaling?
freqs = []
starts = []
for i in range(-30, 31):
starts.append(i)
freqs.append(myDict[i])
# print freqs
fig, ax = plt.subplots()
# plt.set_title('{} codon periodicity'.format(bid))
plt.xlabel("-30 to 30 relative position")
plt.ylabel("Frequency")
plt.bar(starts, freqs)
fig.savefig('{dir}/{bid}_codon_periodicity_plot.png'.format(
dir=newDir, bid=bid))
'''
Picard tools
java -jar picard.jar CollectMultipleMetrics
I=2017-221.uniq_sorted.bam
O= multiple_metrics
R=GRCh37.p13.genome.fa
java -jar picard.jar CollectGcBiasMetrics
I= .uniq
O=gc_bias_metrics.txt
CHART=gc_bias_metrics.pdf
S=summary_metrics.txt
R=reference_sequence.fasta
java -jar picard.jar CollectRnaSeqMetrics
I=input.bam
O=output.RNA_Metrics
REF_FLAT=ref_flat.txt
STRAND=FIRST_READ_TRANSCRIPTION_STRAND
java -jar picard.jar MarkDuplicates
I=input.bam
O=marked_duplicates.bam
M=marked_dup_metrics.txt
ASSUME_SORTED=true
'''
for bid in bid_list:
newDir = new_dir(outputDir, bid)
picard.run(
Parameter('CollectMultipleMetrics'),
Parameter('I={}'.format(bam)), # input
Parameter('O={}/{}.multiple_metrics'.format(newDir,
bid)), # output
Parameter('R={}'.format(pathTo_genomeFasta)) # genomeReference
)
picard.run(
Parameter('CollectGcBiasMetrics'),
Parameter('I={}'.format(bam)), # input
Parameter('O={}/{}.gc_bias_metrics'.format(newDir,
bid)), # output
Parameter('CHART={}/{}.gc_bias_metrics.pdf'.format(
newDir, bid)), # chart
Parameter('S={}/{}.summary_metrics'.format(
newDir, bid)), # summary metrics
Parameter(
'R={}'.format(pathTo_genomeFasta)) # genome reference
)
picard.run(
Parameter('CollectRnaSeqMetrics'),
Parameter('I={}'.format(bam)), # input
Parameter('O={}/{}.RNA_Metrics'.format(newDir, bid)), # output
Parameter('REF_FLAT={}'.format(
'{}'.format(pathTo_ref_flat))), # ref_flat
Parameter(
'STRAND=FIRST_READ_TRANSCRIPTION_STRAND') # strandedness
)
picard.run(
Parameter('MarkDuplicates'),
Parameter('I={}/{}.uniq_sorted.bam'.format(newDir,
bid)), # input
Parameter('O={}/{}.marked_duplicates.bam'.format(
newDir, bid)), # output
Parameter('M={}/{}.marked_dup_metrics.txt'.format(
newDir, bid)), # marked dup metrics
Parameter('ASSUME_SORTED=true') # It is sorted
)
'''
subread: featureCounts
featureCounts -a /path_to_gtf/gencode.v19.annotation.gtf -o <bid>.featureCounts <bid>.uniq_sorted.bam
'''
for bid in bid_list:
newDir = new_dir(outputDir, bid)
featureCounts.run(
Parameter('-a', '{}'.format(pathToGtf)), # gtf
Parameter('-s', '1'), # strand-specific read counting
Parameter('-o', '{}/{}.featureCounts'.format(newDir,
bid)), # output
Parameter('{}/{}.uniq_sorted.bam'.format(newDir, bid)) # input
)
'''
FastQC
fastqc --outdir=/path_to/<bid>/ /path_to_fastq/<bid>.fastq.gz
'''
for fastqlib in fastqFiles:
fastq, bid = fastqlib.split(':')
newDir = new_dir(outputDir, bid)
fastQC.run(
Parameter('--outdir={}'.format(newDir)), # output
Parameter('--t', numThreads),
Parameter(fastq) # input
)
def run_pipeline(self, pipeline_args, pipeline_config):
# Instantiate variable from argparse
read_pairs = pipeline_args['reads']
output_dir = os.path.abspath(pipeline_args['output'])
logs_dir = os.path.join(output_dir, 'logs')
lib_prefix = pipeline_args['lib']
step = int(pipeline_args['step'])
forward_adapter = pipeline_args['forward_adapter']
reverse_adapter = pipeline_args['reverse_adapter']
# Create output, tmp, and logs directories
tmp_dir = os.path.join(output_dir, 'tmp')
subprocess.call(['mkdir', '-p', output_dir, tmp_dir, logs_dir])
#Keep list of items to delete
staging_delete = [tmp_dir]
bwa_bam_outs = []
qc_data = {
'total_raw_reads_counts': [],
'trimmed_reads_counts': [],
'num_reads_mapped': [],
'num_read_removed_steric_hinderence': '0',
'percent_duplicate_reads': '0',
'num_unique_reads_mapped': [], #implemented
'num_mtDNA_reads_mapped': [],
'percent_mtDNA_reads_mapped': '0' ,
'num_reads_mapped_after_filtering': '-1', #TODO This isn't implemented
'num_peaks_called': '-1',
#TODO Get number of peaks in annotation sites
}
#Instatiate software instances
cutadapt = Software('cutadapt', pipeline_config['cutadapt']['path'])
fastqc = Software('FastQC', pipeline_config['fastqc']['path'])
bwa_aln = Software('BWA aln', pipeline_config['bwa']['path'] + ' aln')
bwa_sampe = Software('BWA sampe', pipeline_config['bwa']['path'] + ' sampe')
samtools_view = Software('samtools view', pipeline_config['samtools']['path'] + ' view')
samtools_flagstat = Software('samtools flagstat', pipeline_config['samtools']['path'] + ' flagstat')
samtools_index = Software('samtools index', pipeline_config['samtools']['path'] + ' index')
novosort = Software('novosort', pipeline_config['novosort']['path'])
picard_mark_dup = Software('Picard MarkDuplicates', pipeline_config['picard']['path'] + ' MarkDuplicates')
picard_insert_metrics = Software('Picard CollectInsertSizeMetrics', pipeline_config['picard']['path'] + ' CollectInsertSizeMetrics')
bedtools_bamtobed = Software('bedtools bamtobed', pipeline_config['bedtools']['path'] + ' bamtobed')
bedtools_sort = Software('bedtools sort', pipeline_config['bedtools']['path'] + 'sort')
bedtools_merge = Software('bedtools merge', pipeline_config['bedtools']['path'] + ' merge')
bedtools_intersect = Software('bedtools intersect', pipeline_config['bedtools']['path'] + ' intersect')
macs2_callpeak = Software('macs2 callpeak', pipeline_config['macs2']['path'] + ' callpeak')
if step <= 1:
for i, read_pair in enumerate(read_pairs):
read1, read2 = read_pair.split(':')
#QC: Get raw fastq read counts
qc_data['total_raw_reads_counts'].append([
str(int(self.count_gzipped_lines(read1))/4),
str(int(self.count_gzipped_lines(read2))/4)
])
trimmed_read1_filename = os.path.join(output_dir,
lib_prefix + '_{}_read1.trimmed.fastq.gz'.format(i))
trimmed_read2_filename = os.path.join(output_dir,
lib_prefix + '_{}_read2.trimmed.fastq.gz'.format(i))
cutadapt.run(
Parameter('--quality-base=33'),
Parameter('--minimum-length=5'),
Parameter('-q', '30'), # Minimum quality score
Parameter('--output={}'.format(trimmed_read1_filename)),
Parameter('--paired-output={}'.format(trimmed_read2_filename)),
Parameter('-a', forward_adapter if forward_adapter else 'ZZZ'),
Parameter('-A', reverse_adapter if reverse_adapter else 'ZZZ'),
Parameter(read1),
Parameter(read2),
Redirect(stream=Redirect.STDOUT, dest=os.path.join(logs_dir, 'cutadapt.summary.log'))
)
# QC: Get trimmed fastq read counts
qc_data['trimmed_reads_counts'].append([
str(int(self.count_gzipped_lines(trimmed_read1_filename))/4),
str(int(self.count_gzipped_lines(trimmed_read2_filename))/4)
])
staging_delete.extend([trimmed_read1_filename, trimmed_read2_filename])
read_pairs[i] = ':'.join([trimmed_read1_filename, trimmed_read2_filename])
if step <= 2:
#Make FastQC Directory
fastqc_output_dir = os.path.join(output_dir, 'fastqc')
subprocess.call(['mkdir', '-p', fastqc_output_dir])
for i, read_pair in enumerate(read_pairs):
for read in read_pair.split(':'):
fastqc.run(
Parameter('--outdir={}'.format(fastqc_output_dir)),
Parameter(read)
)
bwa_aln.run(
Parameter('-t', pipeline_config['bwa']['threads']),
Parameter(pipeline_config['bwa']['index-dir']),
Parameter(read),
Redirect(stream=Redirect.STDOUT, dest='{}.sai'.format(read))
)
staging_delete.append('{}.sai'.format(read))
if step <= 3:
for i, read_pair in enumerate(read_pairs):
read1, read2 = read_pair.split(':')
bwa_bam_output = os.path.join(output_dir, '{}.{}.bam'.format(lib_prefix, i))
bwa_sampe.run(
Parameter('-a', '2000'), # Maximum insert size
Parameter('-n', '1'),
Parameter(pipeline_config['bwa']['index-dir']),
Parameter('{}.sai'.format(read1)),
Parameter('{}.sai'.format(read2)),
Parameter(read1),
Parameter(read2),
Redirect(stream=Redirect.STDERR, dest=os.path.join(logs_dir, 'bwa_sampe.log')),
Pipe(
samtools_view.pipe(
Parameter('-hSb'),
Parameter('-o', bwa_bam_output),
Parameter('-') # Get input from stdin
)
)
)
bwa_bam_outs.append(bwa_bam_output)
if step <= 4:
for i, bwa_bam in enumerate(bwa_bam_outs):
samtools_flagstat.run(
Parameter(bwa_bam),
Redirect(stream=Redirect.STDOUT, dest=bwa_bam + '.flagstat')
)
#QC: Get number of mapped reads from this bam
try:
with open(bwa_bam + '.flagstat') as flagstats:
flagstats_contents = flagstats.read()
target_line = re.search(r'(\d+) \+ \d+ mapped', flagstats_contents)
if target_line is not None:
qc_data['num_reads_mapped'].append(str(int(target_line.group(1))/2))
else:
qc_data['num_reads_mapped'].append('0')
except:
qc_data['num_reads_mapped'].append('Could not open flagstats {}'.format(
bwa_bam + '.flagstat'
))
sortmerged_bam = os.path.join(output_dir, '{}.sortmerged_bam'.format(lib_prefix))
steric_filter_bam = os.path.join(output_dir, '{}.steric.bam'.format(lib_prefix))
duprm_bam = os.path.join(output_dir, '{}.duprm.bam'.format(lib_prefix))
unique_bam = os.path.join(output_dir, '{}.unique.bam'.format(lib_prefix))
unmappedrm_bam = os.path.join(output_dir, '{}.unmappedrm.bam'.format(lib_prefix))
chrmrm_bam = os.path.join(output_dir, '{}.chrmrm.bam'.format(lib_prefix))
# binning read based off template size
nucleosome_free_reads = os.path.join(output_dir, '{}.nucleosome_free.bam'.format(lib_prefix))
mononucleosome_reads = os.path.join(output_dir, '{}.mononucleosome.bam'.format(lib_prefix))
dinucleosome_reads = os.path.join(output_dir, '{}.dinucleosome.bam'.format(lib_prefix))
trinucleosome_reads = os.path.join(output_dir, '{}.trinucleosome.bam'.format(lib_prefix))
chrM_bam = os.path.join(output_dir, '{}.chrM.bam'.format(lib_prefix))
novosort.run(
Parameter('--threads', pipeline_config['novosort']['threads']),
Parameter('--tmpcompression', '6'),
Parameter('--tmpdir', tmp_dir),
Parameter(*[bam for bam in bwa_bam_outs]),
Redirect(stream=Redirect.STDOUT, dest=sortmerged_bam),
Redirect(stream=Redirect.STDERR, dest=os.path.join(logs_dir, 'novosort.log'))
)
# This creates a dependency on pysam
# Removes reads with template length < 38 due to steric hinderence
samtools_index.run(Parameter(sortmerged_bam))
sortmerged_bam_alignmentfile = pysam.AlignmentFile(sortmerged_bam, 'rb')
steric_filter_bam_alignmentfile = pysam.AlignmentFile(steric_filter_bam, 'wb',
template=sortmerged_bam_alignmentfile)
num_removed=0
for read in sortmerged_bam_alignmentfile.fetch():
if abs(int(read.template_length)) >= STERIC_HINDRANCE_CUTOFF:
steric_filter_bam_alignmentfile.write(read)
else:
num_removed += 1
qc_data['num_read_removed_steric_hinderence']=str(num_removed)
sortmerged_bam_alignmentfile.close()
steric_filter_bam_alignmentfile.close()
# Mark and remove MarkDuplicates
markduplicates_metrics_filepath = os.path.join(logs_dir, 'mark_dup.metrics')
picard_mark_dup.run(
Parameter('INPUT={}'.format(steric_filter_bam)),
Parameter('OUTPUT={}'.format(duprm_bam)),
Parameter('TMP_DIR={}'.format(tmp_dir)),
Parameter('METRICS_FILE={}'.format(markduplicates_metrics_filepath)),
Parameter('REMOVE_DUPLICATES=true'),
Parameter('VALIDATION_STRINGENCY=LENIENT'),
Redirect(stream=Redirect.BOTH, dest=os.path.join(logs_dir, 'mark_dup.log'))
)
#QC: Get percent MarkDuplicates
try:
with open(markduplicates_metrics_filepath) as markdup_metrics:
for line in markdup_metrics:
if line[FIRST_CHAR] == '#':
continue
record = line.strip().split('\t')
if len(record) == 9:
if re.match(r'\d+', record[7]) is not None:
qc_data['percent_duplicate_reads'] = record[7]
except:
qc_data['percent_duplicate_reads'] = 'Could not open MarkDuplicates metrics'
# Filter down to uniquely mapped reads
samtools_view.run(
Parameter('-b'),
Parameter('-F', '256'),
Parameter('-q', '10'),
Parameter('-o', unique_bam),
Parameter(duprm_bam)
)
# gets statistics on uniquely mapped reads
for i, unique_map in enumerate(unique_bam):
samtools_flagstat.run(
Parameter(unique_bam),
Redirect(stream=Redirect.STDOUT, dest=unique_bam + '.flagstat')
)
#QC: Get number of mapped reads from unique bams
try:
with open(unique_bam + '.flagstat') as flagstats:
unique_flagstats_contents = flagstats.read()
target_line = re.search(r'(\d+) \+ \d+ mapped', unique_flagstats_contents)
if target_line is not None:
qc_data['num_unique_reads_mapped'].append(str(int(target_line.group(1))/2))
else:
qc_data['num_unique_reads_mapped'].append('0')
except:
qc_data['num_unique_reads_mapped'] + '.flagstat'
# make AlignmentFile object to extract binned reads and chrM reads from the unique bam
samtools_index.run(Parameter(unique_bam))
unique_bam_alignmentfile = pysam.AlignmentFile(unique_bam, 'rb')
# Bins reads into 4 categories depending on template length read is derived from:
# 50-115 (nucleosome-free), 180-247 (mononucleosome), 315-473 (dinucleosome), 558-615 (trinucleosome)
nucleosome_free_reads_alignmentfile = pysam.AlignmentFile(nucleosome_free_reads, 'wb',
template=unique_bam_alignmentfile)
mononucleosome_reads_alignmentfile = pysam.AlignmentFile(mononucleosome_reads, 'wb',
template=unique_bam_alignmentfile)
dinucleosome_reads_alignmentfile = pysam.AlignmentFile(dinucleosome_reads, 'wb',
template=unique_bam_alignmentfile)
trinucleosome_reads_alignmentfile = pysam.AlignmentFile(trinucleosome_reads, 'wb',
template=unique_bam_alignmentfile)
# Extract chrM into new BAM
chrM_reads_alignmentfile = pysam.AlignmentFile(chrM_bam, 'wb',
template=unique_bam_alignmentfile)
# Binning of nucleosome reads
for read in unique_bam_alignmentfile.fetch():
if abs(int(read.template_length)) >= 50 and abs(int(read.template_length)) <= 115:
nucleosome_free_reads_alignmentfile.write(read)
elif abs(int(read.template_length)) >= 180 and abs(int(read.template_length)) <= 247:
mononucleosome_reads_alignmentfile.write(read)
elif abs(int(read.template_length)) >= 315 and abs(int(read.template_length)) <= 473:
dinucleosome_reads_alignmentfile.write(read)
elif abs(int(read.template_length)) >= 558 and abs(int(read.template_length)) <= 615:
trinucleosome_reads_alignmentfile.write(read)
else:
continue;
#stores chrM reads in separate file
for read in unique_bam_alignmentfile.fetch():
if read.reference_name == 'chrM':
chrM_reads_alignmentfile.write(read)
nucleosome_free_reads_alignmentfile.close()
mononucleosome_reads_alignmentfile.close()
dinucleosome_reads_alignmentfile.close()
trinucleosome_reads_alignmentfile.close()
chrM_reads_alignmentfile.close()
# gets series of flagstats results for non-main files
samtools_flagstat.run(
Parameter(nucleosome_free_reads),
Redirect(stream=Redirect.STDOUT, dest=nucleosome_free_reads + '.flagstat'))
samtools_flagstat.run(
Parameter(mononucleosome_reads),
Redirect(stream=Redirect.STDOUT, dest=mononucleosome_reads + '.flagstat'))
samtools_flagstat.run(
Parameter(dinucleosome_reads),
Redirect(stream=Redirect.STDOUT, dest=dinucleosome_reads + '.flagstat'))
samtools_flagstat.run(
Parameter(trinucleosome_reads),
Redirect(stream=Redirect.STDOUT, dest=trinucleosome_reads + '.flagstat'))
# gets statistics on chrM mapped reads
samtools_index.run(Parameter(chrM_bam))
for i, chrM_map in enumerate(chrM_bam):
samtools_flagstat.run(
Parameter(chrM_bam),
Redirect(stream=Redirect.STDOUT, dest=chrM_bam + '.flagstat')
)
try:
with open(chrM_bam + '.flagstat') as flagstats:
chrM_flagstats_contents = flagstats.read()
target_line = re.search(r'(\d+) \+ \d+ mapped', chrM_flagstats_contents)
if target_line is not None:
qc_data['num_mtDNA_reads_mapped'].append(str(int(target_line.group(1))/2))
else:
qc_data['num_mtDNA_reads_mapped'].append('0')
except:
qc_data['num_mtDNA_reads_mapped'] + '.flagstat'
# Remove unmapped reads
samtools_view.run(
Parameter('-b'),
Parameter('-F', '12'),
Parameter('-o', unmappedrm_bam),
Parameter(unique_bam)
)
# Create BAM index, then remove chrM
samtools_index.run(
Parameter(unmappedrm_bam)
)
# Remove chrM
all_chr = [Parameter('chr{}'.format(chromosome)) for chromosome in map(str, range(1, 23)) + ['X', 'Y']]
samtools_view.run(
Parameter('-b'),
Parameter('-o', chrmrm_bam),
Parameter(unmappedrm_bam),
*all_chr
)
# Stage delete for temporary files
staging_delete.extend([
sortmerged_bam,
sortmerged_bam + '.bai', # BAM index file
steric_filter_bam,
unique_bam,
duprm_bam,
unmappedrm_bam,
unmappedrm_bam + '.bai', # BAM index file
chrmrm_bam
])
if step <= 5:
# Generate filename for final processed BAM and BED
processed_bam = os.path.join(output_dir, '{}.processed.bam'.format(lib_prefix))
unshifted_bed = os.path.join(output_dir, '{}.unshifted_bed'.format(lib_prefix))
processed_bed = os.path.join(output_dir, '{}.processed.bed'.format(lib_prefix))
# staging_delete.append(unshifted_bed)
# Generate filename for chrM removed BAM
chrmrm_bam = os.path.join(output_dir, '{}.chrmrm.bam'.format(lib_prefix))
# Remove blacklisted genomic regions
bedtools_intersect.run(
Parameter('-v'),
Parameter('-abam', chrmrm_bam),
Parameter('-b', pipeline_config['bedtools']['blacklist-bed']),
Parameter('-f', '0.5'),
Redirect(stream=Redirect.STDOUT, dest=processed_bam)
)
# QC: Generate insert size metrics PDF
picard_insert_metrics.run(
Parameter('INPUT={}'.format(processed_bam)),
Parameter('OUTPUT={}'.format(os.path.join(logs_dir, lib_prefix + '.insertsize.metrics'))),
Parameter('HISTOGRAM_FILE={}'.format(os.path.join(logs_dir, lib_prefix + '.insertsize.pdf')))
)
# Generate index for processed BAM
samtools_index.run(
Parameter(processed_bam)
)
# Convert BAM to BED
bedtools_bamtobed.run(
Parameter('-i', processed_bam),
Redirect(stream=Redirect.STDOUT, dest=unshifted_bed)
)
staging_delete.append(unshifted_bed)
# Shifting + strand by 4 and - strand by -5, according to the ATACseq paper
# This ysed to be bedtools shift, but they are fired
self.shift_reads(
input_bed_filepath=unshifted_bed,
output_bed_filepath=processed_bed,
log_filepath=os.path.join(logs_dir, 'shift_reads.logs'),
genome_sizes_filepath=pipeline_config['bedtools']['genome-sizes'],
minus_strand_shift=MINUS_STRAND_SHIFT,
plus_strand_shift=PLUS_STRAND_SHIFT
)
# Peak-calling; MACS2
if step <= 6:
# for regular peak calling, including narrow, default q-value=0.01
processed_bed = os.path.join(output_dir, '{}.processed.bed'.format(lib_prefix))
macs2_callpeak.run(
Parameter('-t', processed_bed),
Parameter('-f', 'BED'),
Parameter('-g', 'hs'),
Parameter('-n', str(processed_bed) + '_regular_peak_calls'),
Parameter('--nomodel'),
Parameter('--extsize', '200'),
Parameter('--shift', '-100'),
Parameter('-B', '--SPMR'), # Generates pileup tracks, bedgraph, fragment pileup per million reads
Parameter('--call-summits'),
Parameter('--keep-dup', 'all')
)
#for broad peak calling, q-value=0.05 per MACS2 suggestion on broad peaks
macs2_callpeak.run(
Parameter('-t', processed_bed),
Parameter('-f', 'BED'),
Parameter('-g', 'hs'),
Parameter('-n', str(processed_bed) + '_broad_peak_calls'),
Parameter('-q', '0.05'),
Parameter('--nomodel'),
Parameter('--extsize', '200'),
Parameter('--shift', '-100'),
Parameter('--broad'),
Parameter('--keep-dup', 'all')
)
# QC: Output QC data to file
with open(os.path.join(logs_dir, 'qc_metrics.txt'), 'w') as qc_data_file:
qc_data_file.write(str(qc_data) + '\n')
# Delete temporary files
for delete_file in staging_delete:
subprocess.call(['rm', '-rf', delete_file])
def run_pipeline(self, pipeline_args, pipeline_config):
# Instantiate variable from argparse
read_pairs = pipeline_args['reads']
output_dir = os.path.abspath(pipeline_args['output'])
logs_dir = os.path.join(output_dir, 'logs')
lib_prefix = pipeline_args['lib']
step = int(pipeline_args['step'])
forward_adapter = pipeline_args['forward_adapter']
reverse_adapter = pipeline_args['reverse_adapter']
# Create output, tmp, and logs directories
tmp_dir = os.path.join(output_dir, 'tmp')
subprocess.call(['mkdir', '-p', output_dir, tmp_dir, logs_dir])
#Keep list of items to delete
staging_delete = [tmp_dir]
bwa_bam_outs = []
qc_data = {
'total_raw_reads_counts': [],
'trimmed_reads_counts': [],
'num_reads_mapped': [],
'num_read_removed_steric_hinderence': '0',
'percent_duplicate_reads': '0',
'num_unique_reads_mapped': [], #implemented
'num_mtDNA_reads_mapped': [],
'percent_mtDNA_reads_mapped': '0' ,
'num_reads_mapped_after_filtering': '-1', #TODO This isn't implemented
'num_peaks_called': '-1',
#TODO Get number of peaks in annotation sites
}
#Instatiate software instances
cutadapt = Software('cutadapt', pipeline_config['cutadapt']['path'])
fastqc = Software('FastQC', pipeline_config['fastqc']['path'])
bwa_aln = Software('BWA aln', pipeline_config['bwa']['path'] + ' aln')
bwa_sampe = Software('BWA sampe', pipeline_config['bwa']['path'] + ' sampe')
samtools_view = Software('samtools view', pipeline_config['samtools']['path'] + ' view')
samtools_flagstat = Software('samtools flagstat', pipeline_config['samtools']['path'] + ' flagstat')
samtools_index = Software('samtools index', pipeline_config['samtools']['path'] + ' index')
samtools_sort = Software('samtools sort', pipeline_config['samtools']['path'] + ' sort')
novosort = Software('novosort', pipeline_config['novosort']['path'])
picard_mark_dup = Software('Picard MarkDuplicates', pipeline_config['picard']['path'] + ' MarkDuplicates')
picard_insert_metrics = Software('Picard CollectInsertSizeMetrics', pipeline_config['picard']['path'] + ' CollectInsertSizeMetrics')
bedtools_bamtobed = Software('bedtools bamtobed', pipeline_config['bedtools']['path'] + ' bamtobed')
bedtools_sort = Software('bedtools sort', pipeline_config['bedtools']['path'] + 'sort')
bedtools_merge = Software('bedtools merge', pipeline_config['bedtools']['path'] + ' merge')
bedtools_intersect = Software('bedtools intersect', pipeline_config['bedtools']['path'] + ' intersect')
macs2_callpeak = Software('macs2 callpeak', pipeline_config['macs2']['path'] + ' callpeak')
if step <= 1:
for i, read_pair in enumerate(read_pairs):
read1, read2 = read_pair.split(':')
#QC: Get raw fastq read counts
qc_data['total_raw_reads_counts'].append([
str(int(self.count_gzipped_lines(read1))/4),
str(int(self.count_gzipped_lines(read2))/4)
])
trimmed_read1_filename = os.path.join(output_dir,
lib_prefix + '_{}_read1.trimmed.fastq.gz'.format(i))
trimmed_read2_filename = os.path.join(output_dir,
lib_prefix + '_{}_read2.trimmed.fastq.gz'.format(i))
cutadapt.run(
Parameter('--quality-base=33'),
Parameter('--minimum-length=5'),
Parameter('-q', '30'), # Minimum quality score
Parameter('--output={}'.format(trimmed_read1_filename)),
Parameter('--paired-output={}'.format(trimmed_read2_filename)),
Parameter('-a', forward_adapter if forward_adapter else 'ZZZ'),
Parameter('-A', reverse_adapter if reverse_adapter else 'ZZZ'),
Parameter(read1),
Parameter(read2),
Redirect(stream=Redirect.STDOUT, dest=os.path.join(logs_dir, 'cutadapt.summary.log'))
)
# QC: Get trimmed fastq read counts
qc_data['trimmed_reads_counts'].append([
str(int(self.count_gzipped_lines(trimmed_read1_filename))/4),
str(int(self.count_gzipped_lines(trimmed_read2_filename))/4)
])
staging_delete.extend([trimmed_read1_filename, trimmed_read2_filename])
read_pairs[i] = ':'.join([trimmed_read1_filename, trimmed_read2_filename])
if step <= 2:
#Make FastQC Directory
fastqc_output_dir = os.path.join(output_dir, 'fastqc')
subprocess.call(['mkdir', '-p', fastqc_output_dir])
for i, read_pair in enumerate(read_pairs):
for read in read_pair.split(':'):
fastqc.run(
Parameter('--outdir={}'.format(fastqc_output_dir)),
Parameter(read)
)
bwa_aln.run(
Parameter('-t', pipeline_config['bwa']['threads']),
Parameter(pipeline_config['bwa']['index-dir']),
Parameter(read),
Redirect(stream=Redirect.STDOUT, dest='{}.sai'.format(read))
)
staging_delete.append('{}.sai'.format(read))
if step <= 3:
for i, read_pair in enumerate(read_pairs):
read1, read2 = read_pair.split(':')
bwa_bam_output = os.path.join(output_dir, '{}.{}.bam'.format(lib_prefix, i))
bwa_sampe.run(
Parameter('-a', '2000'), # Maximum insert size
Parameter('-n', '1'),
Parameter(pipeline_config['bwa']['index-dir']),
Parameter('{}.sai'.format(read1)),
Parameter('{}.sai'.format(read2)),
Parameter(read1),
Parameter(read2),
Redirect(stream=Redirect.STDERR, dest=os.path.join(logs_dir, 'bwa_sampe.log')),
Pipe(
samtools_view.pipe(
Parameter('-hSb'),
Parameter('-o', bwa_bam_output),
Parameter('-') # Get input from stdin
)
)
)
bwa_bam_outs.append(bwa_bam_output)
if step <= 4:
for i, bwa_bam in enumerate(bwa_bam_outs):
samtools_flagstat.run(
Parameter(bwa_bam),
Redirect(stream=Redirect.STDOUT, dest=bwa_bam + '.flagstat')
)
#QC: Get number of mapped reads from this bam
try:
with open(bwa_bam + '.flagstat') as flagstats:
flagstats_contents = flagstats.read()
target_line = re.search(r'(\d+) \+ \d+ mapped', flagstats_contents)
if target_line is not None:
qc_data['num_reads_mapped'].append(str(int(target_line.group(1))/2))
else:
qc_data['num_reads_mapped'].append('0')
except:
qc_data['num_reads_mapped'].append('Could not open flagstats {}'.format(
bwa_bam + '.flagstat'
))
sortmerged_bam = os.path.join(output_dir, '{}.sortmerged_bam'.format(lib_prefix))
steric_filter_bam = os.path.join(output_dir, '{}.steric.bam'.format(lib_prefix))
duprm_bam = os.path.join(output_dir, '{}.duprm.bam'.format(lib_prefix))
unique_bam = os.path.join(output_dir, '{}.unique.bam'.format(lib_prefix))
unmappedrm_bam = os.path.join(output_dir, '{}.unmappedrm.bam'.format(lib_prefix))
chrmrm_bam = os.path.join(output_dir, '{}.chrmrm.bam'.format(lib_prefix))
# binning read based off template size
nucleosome_free_reads = os.path.join(output_dir, '{}.nucleosome_free.bam'.format(lib_prefix))
mononucleosome_reads = os.path.join(output_dir, '{}.mononucleosome.bam'.format(lib_prefix))
dinucleosome_reads = os.path.join(output_dir, '{}.dinucleosome.bam'.format(lib_prefix))
trinucleosome_reads = os.path.join(output_dir, '{}.trinucleosome.bam'.format(lib_prefix))
chrM_bam = os.path.join(output_dir, '{}.chrM.bam'.format(lib_prefix))
sorted_for_PE_bam = os.path.join(output_dir, '{}.sorted_for_PE'.format(lib_prefix))
novosort.run(
Parameter('--threads', pipeline_config['novosort']['threads']),
Parameter('--tmpcompression', '6'),
Parameter('--tmpdir', tmp_dir),
Parameter(*[bam for bam in bwa_bam_outs]),
Redirect(stream=Redirect.STDOUT, dest=sortmerged_bam),
Redirect(stream=Redirect.STDERR, dest=os.path.join(logs_dir, 'novosort.log'))
)
# This creates a dependency on pysam
# Removes reads with template length < 38 due to steric hinderence
samtools_index.run(Parameter(sortmerged_bam))
sortmerged_bam_alignmentfile = pysam.AlignmentFile(sortmerged_bam, 'rb')
steric_filter_bam_alignmentfile = pysam.AlignmentFile(steric_filter_bam, 'wb',
template=sortmerged_bam_alignmentfile)
num_removed=0
for read in sortmerged_bam_alignmentfile.fetch():
if abs(int(read.template_length)) >= STERIC_HINDRANCE_CUTOFF:
steric_filter_bam_alignmentfile.write(read)
else:
num_removed += 1
qc_data['num_read_removed_steric_hinderence']=str(num_removed)
sortmerged_bam_alignmentfile.close()
steric_filter_bam_alignmentfile.close()
# Mark and remove MarkDuplicates
markduplicates_metrics_filepath = os.path.join(logs_dir, 'mark_dup.metrics')
picard_mark_dup.run(
Parameter('INPUT={}'.format(steric_filter_bam)),
Parameter('OUTPUT={}'.format(duprm_bam)),
Parameter('TMP_DIR={}'.format(tmp_dir)),
Parameter('METRICS_FILE={}'.format(markduplicates_metrics_filepath)),
Parameter('REMOVE_DUPLICATES=true'),
Parameter('VALIDATION_STRINGENCY=LENIENT'),
Redirect(stream=Redirect.BOTH, dest=os.path.join(logs_dir, 'mark_dup.log'))
)
#QC: Get percent MarkDuplicates
try:
with open(markduplicates_metrics_filepath) as markdup_metrics:
for line in markdup_metrics:
if line[FIRST_CHAR] == '#':
continue
record = line.strip().split('\t')
if len(record) == 9:
if re.match(r'\d+', record[7]) is not None:
qc_data['percent_duplicate_reads'] = record[7]
except:
qc_data['percent_duplicate_reads'] = 'Could not open MarkDuplicates metrics'
# Filter down to uniquely mapped reads
samtools_view.run(
Parameter('-b'),
Parameter('-F', '256'),
Parameter('-q', '10'),
Parameter('-o', unique_bam),
Parameter(duprm_bam)
)
# gets statistics on uniquely mapped reads
for i, unique_map in enumerate(unique_bam):
samtools_flagstat.run(
Parameter(unique_bam),
Redirect(stream=Redirect.STDOUT, dest=unique_bam + '.flagstat')
)
#QC: Get number of mapped reads from unique bams
try:
with open(unique_bam + '.flagstat') as flagstats:
unique_flagstats_contents = flagstats.read()
target_line = re.search(r'(\d+) \+ \d+ mapped', unique_flagstats_contents)
if target_line is not None:
qc_data['num_unique_reads_mapped'].append(str(int(target_line.group(1))/2))
else:
qc_data['num_unique_reads_mapped'].append('0')
except:
qc_data['num_unique_reads_mapped'] + '.flagstat'
# make AlignmentFile object to extract binned reads and chrM reads from the unique bam
samtools_index.run(Parameter(unique_bam))
unique_bam_alignmentfile = pysam.AlignmentFile(unique_bam, 'rb')
# Bins reads into 4 categories depending on template length read is derived from:
# 50-115 (nucleosome-free), 180-247 (mononucleosome), 315-473 (dinucleosome), 558-615 (trinucleosome)
nucleosome_free_reads_alignmentfile = pysam.AlignmentFile(nucleosome_free_reads, 'wb',
template=unique_bam_alignmentfile)
mononucleosome_reads_alignmentfile = pysam.AlignmentFile(mononucleosome_reads, 'wb',
template=unique_bam_alignmentfile)
dinucleosome_reads_alignmentfile = pysam.AlignmentFile(dinucleosome_reads, 'wb',
template=unique_bam_alignmentfile)
trinucleosome_reads_alignmentfile = pysam.AlignmentFile(trinucleosome_reads, 'wb',
template=unique_bam_alignmentfile)
# Extract chrM into new BAM
chrM_reads_alignmentfile = pysam.AlignmentFile(chrM_bam, 'wb',
template=unique_bam_alignmentfile)
# Binning of nucleosome reads
for read in unique_bam_alignmentfile.fetch():
if abs(int(read.template_length)) >= 50 and abs(int(read.template_length)) <= 115:
nucleosome_free_reads_alignmentfile.write(read)
elif abs(int(read.template_length)) >= 180 and abs(int(read.template_length)) <= 247:
mononucleosome_reads_alignmentfile.write(read)
elif abs(int(read.template_length)) >= 315 and abs(int(read.template_length)) <= 473:
dinucleosome_reads_alignmentfile.write(read)
elif abs(int(read.template_length)) >= 558 and abs(int(read.template_length)) <= 615:
trinucleosome_reads_alignmentfile.write(read)
else:
continue;
#stores chrM reads in separate file
for read in unique_bam_alignmentfile.fetch():
if read.reference_name == 'chrM':
chrM_reads_alignmentfile.write(read)
nucleosome_free_reads_alignmentfile.close()
mononucleosome_reads_alignmentfile.close()
dinucleosome_reads_alignmentfile.close()
trinucleosome_reads_alignmentfile.close()
chrM_reads_alignmentfile.close()
# gets series of flagstats results for non-main files
samtools_flagstat.run(
Parameter(nucleosome_free_reads),
Redirect(stream=Redirect.STDOUT, dest=nucleosome_free_reads + '.flagstat'))
samtools_flagstat.run(
Parameter(mononucleosome_reads),
Redirect(stream=Redirect.STDOUT, dest=mononucleosome_reads + '.flagstat'))
samtools_flagstat.run(
Parameter(dinucleosome_reads),
Redirect(stream=Redirect.STDOUT, dest=dinucleosome_reads + '.flagstat'))
samtools_flagstat.run(
Parameter(trinucleosome_reads),
Redirect(stream=Redirect.STDOUT, dest=trinucleosome_reads + '.flagstat'))
# gets statistics on chrM mapped reads
samtools_index.run(Parameter(chrM_bam))
for i, chrM_map in enumerate(chrM_bam):
samtools_flagstat.run(
Parameter(chrM_bam),
Redirect(stream=Redirect.STDOUT, dest=chrM_bam + '.flagstat')
)
try:
with open(chrM_bam + '.flagstat') as flagstats:
chrM_flagstats_contents = flagstats.read()
target_line = re.search(r'(\d+) \+ \d+ mapped', chrM_flagstats_contents)
if target_line is not None:
qc_data['num_mtDNA_reads_mapped'].append(str(int(target_line.group(1))/2))
else:
qc_data['num_mtDNA_reads_mapped'].append('0')
except:
qc_data['num_mtDNA_reads_mapped'] + '.flagstat'
# Remove unmapped reads
samtools_view.run(
Parameter('-b'),
Parameter('-F', '12'),
Parameter('-o', unmappedrm_bam),
Parameter(unique_bam)
)
# Create BAM index, then remove chrM
samtools_index.run(
Parameter(unmappedrm_bam)
)
# Remove chrM
all_chr = [Parameter('chr{}'.format(chromosome)) for chromosome in map(str, range(1, 23)) + ['X', 'Y']]
samtools_view.run(
Parameter('-b'),
Parameter('-o', chrmrm_bam),
Parameter(unmappedrm_bam),
*all_chr
)
# Stage delete for temporary files
staging_delete.extend([
sortmerged_bam,
sortmerged_bam + '.bai', # BAM index file
steric_filter_bam,
unique_bam,
duprm_bam,
unmappedrm_bam,
unmappedrm_bam + '.bai', # BAM index file
chrmrm_bam
])
if step <= 5:
# Generate filename for final processed BAM and BED
processed_bam = os.path.join(output_dir, '{}.processed.bam'.format(lib_prefix))
unshifted_bed = os.path.join(output_dir, '{}.unshifted_bed'.format(lib_prefix))
processed_bed = os.path.join(output_dir, '{}.processed.bed'.format(lib_prefix))
unshifted_bedpe = os.path.join(output_dir, '{}.unshifted_bedpe'.format(lib_prefix))
processed_bedpe_to_bed = os.path.join(output_dir,'{}.processed_bedpe_to_bed'.format(lib_prefix))
# staging_delete.append(unshifted_bed)
# Generate filename for chrM removed BAM
chrmrm_bam = os.path.join(output_dir, '{}.chrmrm.bam'.format(lib_prefix))
# Remove blacklisted genomic regions
bedtools_intersect.run(
Parameter('-v'),
Parameter('-abam', chrmrm_bam),
Parameter('-b', pipeline_config['bedtools']['blacklist-bed']),
Parameter('-f', '0.5'),
Redirect(stream=Redirect.STDOUT, dest=processed_bam)
)
# QC: Generate insert size metrics PDF
picard_insert_metrics.run(
Parameter('INPUT={}'.format(processed_bam)),
Parameter('OUTPUT={}'.format(os.path.join(logs_dir, lib_prefix + '.insertsize.metrics'))),
Parameter('HISTOGRAM_FILE={}'.format(os.path.join(logs_dir, lib_prefix + '.insertsize.pdf')))
)
# Generate index for processed BAM
samtools_index.run(
Parameter(processed_bam)
)
# Convert BAM to BED
bedtools_bamtobed.run(
Parameter('-i', processed_bam),
Redirect(stream=Redirect.STDOUT, dest=unshifted_bed)
)
# Convert BAM to BEDPE, with specific quality and only properly paired reads, sorted by name
samtools_view.run(
Parameter('-uf', '0x2'),
Parameter('-F', '1548'),
Parameter('-q', '30'),
Parameter(processed_bam),
Pipe(
samtools_sort.pipe(
Parameter('-n'),
Parameter('-'),
Parameter(sorted_for_PE_bam)
)
)
)
# convert bam to BEDPE
bedtools_bamtobed.run(
Parameter('-i', str(sorted_for_PE_bam)+'.bam'),
Parameter('-bedpe'),
Redirect(stream=Redirect.STDOUT, dest=unshifted_bedpe)
)
unshifted_bedpe_to_bed = open(output_dir+'/'+'{}.unshifted_bedpe_to_bed'.format(lib_prefix), 'w')
with open(unshifted_bedpe) as convertToBed:
for line in convertToBed:
chrpos1, start1, end1, chrpos2, start2, end2, name, score, strand1, strand2=line.split('\t')
bedformat=[chrpos1, start1, end2, name, score, strand1, strand2.rstrip('\n')]
unshifted_bedpe_to_bed.write('\t'.join(bedformat)+'\n')
staging_delete.append(unshifted_bed)
staging_delete.append(output_dir+'/'+'{}.unshifted_bedpe_to_bed'.format(lib_prefix))
# Shifting + strand by 4 and - strand by -5, according to the ATACseq paper
# This ysed to be bedtools shift, but they are fired
self.shift_reads(
input_bed_filepath=unshifted_bed,
output_bed_filepath=processed_bed,
log_filepath=os.path.join(logs_dir, 'shift_reads.logs'),
genome_sizes_filepath=pipeline_config['bedtools']['genome-sizes'],
minus_strand_shift=MINUS_STRAND_SHIFT,
plus_strand_shift=PLUS_STRAND_SHIFT
)
##TO DO, needs modification for bedpe format
self.shift_reads_bedpe(
input_bed_filepath=output_dir+'/'+'{}.unshifted_bedpe_to_bed'.format(lib_prefix),
output_bed_filepath=processed_bedpe_to_bed,
log_filepath=os.path.join(logs_dir, 'shift_reads_bedpe_to_bed.logs'),
genome_sizes_filepath=pipeline_config['bedtools']['genome-sizes'],
minus_strand_shift=MINUS_STRAND_SHIFT,
plus_strand_shift=PLUS_STRAND_SHIFT
)
# Peak-calling; MACS2
if step <= 6:
# for regular peak calling, including narrow, default q-value=0.01
macs2_callpeak.run(
Parameter('-t', processed_bed),
Parameter('-f', 'BED'),
Parameter('-g', 'hs'),
Parameter('-n', str(processed_bed) + '_regular_peak_calls'),
Parameter('--nomodel'),
Parameter('--extsize', '200'),
Parameter('--shift', '-100'),
Parameter('-B', '--SPMR'), # Generates pileup tracks, bedgraph, fragment pileup per million reads
Parameter('--call-summits'),
Parameter('--keep-dup', 'all')
)
#for broad peak calling, q-value=0.05 per MACS2 suggestion on broad peaks
macs2_callpeak.run(
Parameter('-t', processed_bed),
Parameter('-f', 'BED'),
Parameter('-g', 'hs'),
Parameter('-n', str(processed_bed) + '_broad_peak_calls'),
Parameter('-q', '0.05'),
Parameter('--nomodel'),
Parameter('--extsize', '200'),
Parameter('--shift', '-100'),
Parameter('--broad'),
Parameter('--keep-dup', 'all')
)
# for regular peak calling, including narrow, default q-value=0.01 for processed bedpe to bed file
# NOTE: BEDPE for MACS2 is not the same format at BEDPE accepted by NGS/UCSC standards
macs2_callpeak.run(
Parameter('-t', processed_bedpe_to_bed),
Parameter('-f', 'BEDPE'),
Parameter('-g', 'hs'),
Parameter('-n', str(processed_bedpe_to_bed) + '_regular_peak_calls'),
Parameter('--nomodel'),
Parameter('--extsize', '200'),
Parameter('--shift', '-100'),
Parameter('-B', '--SPMR'), # Generates pileup tracks, bedgraph, fragment pileup per million reads
Parameter('--call-summits'),
Parameter('--keep-dup', 'all')
)
#for broad peak calling, q-value=0.05 per MACS2 suggestion on broad peaks for processed bedpe to bed file
# NOTE: BEDPE for MACS2 is not the same format at BEDPE accepted by NGS/UCSC standards
macs2_callpeak.run(
Parameter('-t', processed_bedpe_to_bed),
Parameter('-f', 'BEDPE'),
Parameter('-g', 'hs'),
Parameter('-n', str(processed_bedpe_to_bed) + '_broad_peak_calls'),
Parameter('-q', '0.05'),
Parameter('--nomodel'),
Parameter('--extsize', '200'),
Parameter('--shift', '-100'),
Parameter('--broad'),
Parameter('--keep-dup', 'all')
)
# QC: Output QC data to file
with open(os.path.join(logs_dir, 'qc_metrics.txt'), 'w') as qc_data_file:
qc_data_file.write(str(qc_data) + '\n')
# Delete temporary files
for delete_file in staging_delete:
subprocess.call(['rm', '-rf', delete_file])
