def pbc_qc_se(bam, mito_chr_name, out_dir):
prefix = os.path.join(out_dir,
os.path.basename(strip_ext_bam(bam)))
# strip extension appended in the previous step
prefix = strip_ext(prefix, 'dupmark')
pbc_qc = '{}.lib_complexity.qc'.format(prefix)
cmd2 = 'bedtools bamtobed -i {} | '
cmd2 += 'awk \'BEGIN{{OFS="\\t"}}{{print $1,$2,$3,$6}}\' | '
cmd2 += 'grep -v "^{}\\s" | sort | uniq -c | '
cmd2 += 'awk \'BEGIN{{mt=0;m0=0;m1=0;m2=0}} ($1==1){{m1=m1+1}} '
cmd2 += '($1==2){{m2=m2+1}} {{m0=m0+1}} '
cmd2 += '{{mt=mt+$1}} END{{m1_m2=-1.0; '
cmd2 += 'if(m2>0) m1_m2=m1/m2; m0_mt=0; '
cmd2 += 'if (mt>0) m0_mt=m0/mt; m1_m0=0; if (m0>0) m1_m0=m1/m0; '
cmd2 += 'printf "%d\\t%d\\t%d\\t%d\\t%f\\t%f\\t%f\\n",'
cmd2 += 'mt,m0,m1,m2,m0_mt,m1_m0,m1_m2}}\' > {}'
cmd2 = cmd2.format(
bam,
mito_chr_name,
pbc_qc)
run_shell_cmd(cmd2)
return pbc_qc
def trim_adapter_pe(fastq1, fastq2, adapter1, adapter2, adapter_for_all,
cutadapt_param, out_dir):
if adapter1 and adapter2:
prefix1 = os.path.join(out_dir,
os.path.basename(strip_ext_fastq(fastq1)))
prefix2 = os.path.join(out_dir,
os.path.basename(strip_ext_fastq(fastq2)))
trimmed1 = '{}.trim.fastq.gz'.format(prefix1)
trimmed2 = '{}.trim.fastq.gz'.format(prefix2)
cmd = 'cutadapt {} -a {} -A {} {} {} -o {} -p {}'.format(
cutadapt_param, adapter_for_all if adapter_for_all else adapter1,
adapter_for_all if adapter_for_all else adapter2, fastq1, fastq2,
trimmed1, trimmed2)
run_shell_cmd(cmd)
return [trimmed1, trimmed2]
else:
# make hard link
linked1 = os.path.join(out_dir, os.path.basename(fastq1))
linked2 = os.path.join(out_dir, os.path.basename(fastq2))
os.link(fastq1, linked1)
os.link(fastq2, linked2)
return [linked1, linked2]
def bowtie2_se(fastq, ref_index_prefix,
multimapping, nth, out_dir):
basename = os.path.basename(strip_ext_fastq(fastq))
prefix = os.path.join(out_dir, basename)
bam = '{}.bam'.format(prefix)
align_log = '{}.align.log'.format(prefix)
cmd = 'bowtie2 {} --mm --threads {} -x {} -U {} 2> {} '
cmd += '| samtools view -Su /dev/stdin '
cmd += '| samtools sort /dev/stdin -o {} -T {}'
cmd = cmd.format(
'-k {}'.format(multimapping+1) if multimapping else '',
nth,
ref_index_prefix,
fastq,
align_log,
bam,
prefix)
run_shell_cmd(cmd)
cmd2 = 'cat {}'.format(align_log)
run_shell_cmd(cmd2)
return bam, align_log
def subsample_ta_se(ta, subsample, non_mito, mito_chr_name, out_dir):
prefix = os.path.join(out_dir, os.path.basename(strip_ext_ta(ta)))
ta_subsampled = '{}.{}{}tagAlign.gz'.format(
prefix, 'no_chrM.' if non_mito else '', '{}.'.format(
human_readable_number(subsample)) if subsample > 0 else '')
# bash-only
cmd = 'zcat -f {} | '
if non_mito:
# cmd += 'awk \'{{if ($1!="'+mito_chr_name+'") print $0}}\' | '
cmd += 'grep -v \'^' + mito_chr_name + '\\b\' | '
if subsample > 0:
cmd += 'shuf -n {} --random-source=<(openssl enc -aes-256-ctr '
cmd += '-pass pass:$(zcat -f {} | wc -c) -nosalt '
cmd += '</dev/zero 2>/dev/null) | '
cmd += 'gzip -nc > {}'
cmd = cmd.format(ta, subsample, ta, ta_subsampled)
else:
cmd += 'gzip -nc > {}'
cmd = cmd.format(ta, ta_subsampled)
run_shell_cmd(cmd)
return ta_subsampled
def spr_pe(ta, pseudoreplication_random_seed, out_dir):
prefix = os.path.join(out_dir, os.path.basename(strip_ext_ta(ta)))
tmp_pr1 = '{}.00'.format(prefix)
tmp_pr2 = '{}.01'.format(prefix)
ta_pr1 = '{}.pr1.tagAlign.gz'.format(prefix)
ta_pr2 = '{}.pr2.tagAlign.gz'.format(prefix)
nlines = int((get_num_lines(ta) / 2 + 1) / 2)
if pseudoreplication_random_seed == 0:
random_seed = run_shell_cmd('zcat -f {ta} | wc -c'.format(ta=ta))
log.info(
'Using input file\'s size {random_seed} as random seed for pseudoreplication.'
.format(random_seed=random_seed, ))
else:
random_seed = pseudoreplication_random_seed
log.info(
'Using a fixed integer {random_seed} as random seed for pseudoreplication.'
.format(random_seed=random_seed, ))
# bash-only
run_shell_cmd('zcat -f {ta} | sed \'N;s/\\n/\\t/\' | '
'shuf --random-source=<(openssl enc -aes-256-ctr '
'-pass pass:{random_seed} -nosalt </dev/zero 2>/dev/null) | '
'split -d -l {nlines} - {prefix}.'.format(
ta=ta,
random_seed=random_seed,
nlines=nlines,
prefix=prefix,
))
run_shell_cmd('zcat -f {tmp_pr1} | '
'awk \'BEGIN{{OFS="\\t"}} '
'{{printf "%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n'
'%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n",'
'$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12}}\' | '
'gzip -nc > {ta_pr1}'.format(
tmp_pr1=tmp_pr1,
ta_pr1=ta_pr1,
))
run_shell_cmd('zcat -f {tmp_pr2} | '
'awk \'BEGIN{{OFS="\\t"}} '
'{{printf "%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n'
'%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n",'
'$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12}}\' | '
'gzip -nc > {ta_pr2}'.format(
tmp_pr2=tmp_pr2,
ta_pr2=ta_pr2,
))
rm_f([tmp_pr1, tmp_pr2])
return ta_pr1, ta_pr2
def idr(basename_prefix, peak1, peak2, peak_pooled, peak_type, chrsz, thresh,
rank, out_dir):
prefix = os.path.join(out_dir, basename_prefix)
prefix += '.idr{}'.format(thresh)
idr_peak = '{}.{}.gz'.format(prefix, peak_type)
idr_plot = '{}.unthresholded-peaks.txt.png'.format(prefix)
idr_stdout = '{}.log'.format(prefix)
# temporary
idr_12col_bed = '{}.12-col.bed.gz'.format(peak_type)
idr_out = '{}.unthresholded-peaks.txt'.format(prefix)
idr_tmp = '{}.unthresholded-peaks.txt.tmp'.format(prefix)
idr_out_gz = '{}.unthresholded-peaks.txt.gz'.format(prefix)
cmd1 = 'idr --samples {} {} --peak-list {} --input-file-type narrowPeak '
cmd1 += '--output-file {} --rank {} --soft-idr-threshold {} '
cmd1 += '--plot --use-best-multisummit-IDR --log-output-file {}'
cmd1 = cmd1.format(peak1, peak2, peak_pooled, idr_out, rank, thresh,
idr_stdout)
run_shell_cmd(cmd1)
# clip peaks between 0-chromSize.
bed_clip(idr_out, chrsz, idr_tmp, no_gz=True)
col = get_npeak_col_by_rank(rank)
neg_log10_thresh = -math.log10(thresh)
# LC_COLLATE=C
cmd2 = 'awk \'BEGIN{{OFS="\\t"}} $12>={} '
cmd2 += '{{if ($2<0) $2=0; '
cmd2 += 'print $1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12}}\' {} '
cmd2 += '| sort | uniq | sort -grk{},{} | gzip -nc > {}'
cmd2 = cmd2.format(neg_log10_thresh, idr_tmp, col, col, idr_12col_bed)
run_shell_cmd(cmd2)
cmd3 = 'zcat {} | '
cmd3 += 'awk \'BEGIN{{OFS="\\t"}} '
cmd3 += '{{print $1,$2,$3,$4,$5,$6,$7,$8,$9,$10}}\' | '
cmd3 += 'gzip -nc > {}'
cmd3 = cmd3.format(idr_12col_bed, idr_peak)
run_shell_cmd(cmd3)
cmd4 = 'cat {} | gzip -nc > {}'.format(idr_tmp, idr_out_gz)
run_shell_cmd(cmd4)
rm_f([idr_out, idr_tmp, idr_12col_bed])
rm_f('{}.*.noalternatesummitpeaks.png'.format(prefix))
return idr_peak, idr_plot, idr_out_gz, idr_stdout
def macs2(ta, ctl_ta, chrsz, gensz, pval_thresh, shift, fraglen, cap_num_peak,
ctl_subsample, ctl_paired_end, out_dir):
basename_ta = os.path.basename(strip_ext_ta(ta))
if ctl_ta:
if ctl_subsample:
if ctl_paired_end:
ctl_ta = subsample_ta_pe(ctl_ta,
ctl_subsample,
non_mito=False,
mito_chr_name=None,
r1_only=False,
out_dir=out_dir)
else:
ctl_ta = subsample_ta_se(ctl_ta,
ctl_subsample,
non_mito=False,
mito_chr_name=None,
out_dir=out_dir)
basename_ctl_ta = os.path.basename(strip_ext_ta(ctl_ta))
basename_prefix = '{}_x_{}'.format(basename_ta, basename_ctl_ta)
if len(basename_prefix) > 200: # UNIX cannot have len(filename) > 255
basename_prefix = '{}_x_control'.format(basename_ta)
else:
basename_prefix = basename_ta
prefix = os.path.join(out_dir, basename_prefix)
npeak = '{}.{}.{}.narrowPeak.gz'.format(
prefix, 'pval{}'.format(pval_thresh),
human_readable_number(cap_num_peak))
npeak_tmp = '{}.tmp'.format(npeak)
temp_files = []
cmd0 = ' macs2 callpeak '
cmd0 += '-t {} {} -f BED -n {} -g {} -p {} '
cmd0 += '--nomodel --shift {} --extsize {} --keep-dup all -B --SPMR'
cmd0 = cmd0.format(ta, '-c {}'.format(ctl_ta) if ctl_ta else '', prefix,
gensz, pval_thresh, 0, fraglen)
run_shell_cmd(cmd0)
cmd1 = 'LC_COLLATE=C sort -k 8gr,8gr "{}"_peaks.narrowPeak | '
cmd1 += 'awk \'BEGIN{{OFS="\\t"}}'
cmd1 += '{{$4="Peak_"NR; if ($2<0) $2=0; if ($3<0) $3=0; if ($10==-1) '
cmd1 += '$10=$2+int(($3-$2+1)/2.0); print $0}}\' > {}'
cmd1 = cmd1.format(prefix, npeak_tmp)
run_shell_cmd(cmd1)
cmd2 = 'head -n {} {} | gzip -nc > {}'.format(cap_num_peak, npeak_tmp,
npeak)
run_shell_cmd(cmd2)
rm_f(npeak_tmp)
# remove temporary files
temp_files.append("{}_*".format(prefix))
rm_f(temp_files)
return npeak
def bam2ta_pe(bam, nth, out_dir):
prefix = os.path.join(out_dir, os.path.basename(strip_ext_bam(bam)))
ta = '{}.tagAlign.gz'.format(prefix)
# intermediate files
bedpe = '{}.bedpe.gz'.format(prefix)
nmsrt_bam = samtools_name_sort(bam, nth, out_dir)
cmd1 = 'LC_COLLATE=C bedtools bamtobed -bedpe -mate1 -i {} | '
# cmd1 += 'sort -k1,1 -k2,2n -k3,3n | '
cmd1 += 'gzip -nc > {}'
cmd1 = cmd1.format(nmsrt_bam, bedpe)
run_shell_cmd(cmd1)
rm_f(nmsrt_bam)
cmd2 = 'zcat -f {} | '
cmd2 += 'awk \'BEGIN{{OFS="\\t"}}'
cmd2 += '{{printf "%s\\t%s\\t%s\\tN\\t1000\\t%s\\n'
cmd2 += '%s\\t%s\\t%s\\tN\\t1000\\t%s\\n",'
cmd2 += '$1,$2,$3,$9,$4,$5,$6,$10}}\' | '
cmd2 += 'gzip -nc > {}'
cmd2 = cmd2.format(bedpe, ta)
run_shell_cmd(cmd2)
rm_f(bedpe)
return ta
def get_fract_reads_in_regions(reads_bed, regions_bed):
"""Function that takes in bed file of reads and bed file of regions and
gets fraction of reads sitting in said regions
"""
# uses new run_shell_cmd
cmd = "bedtools sort -i {} | "
cmd += "bedtools merge -i stdin | "
cmd += "bedtools intersect -u -nonamecheck -a {} -b stdin | "
cmd += "wc -l"
cmd = cmd.format(regions_bed, reads_bed)
intersect_read_count = int(run_shell_cmd(cmd))
total_read_count = get_num_lines(reads_bed)
fract_reads = float(intersect_read_count) / total_read_count
return intersect_read_count, fract_reads
def main():
# read params
args = parse_arguments()
log.info('Initializing and making output directory...')
mkdir_p(args.out_dir)
# declare temp arrays
temp_files = [] # files to deleted later at the end
# if bowtie2 index is tarball then unpack it
if args.bowtie2_index_prefix_or_tar.endswith('.tar') or \
args.bowtie2_index_prefix_or_tar.endswith('.tar.gz'):
log.info('Unpacking bowtie2 index tar...')
tar = args.bowtie2_index_prefix_or_tar
# untar
untar(tar, args.out_dir)
bowtie2_index_prefix = find_bowtie2_index_prefix(args.out_dir)
temp_files.append('{}*'.format(bowtie2_index_prefix))
else:
bowtie2_index_prefix = args.bowtie2_index_prefix_or_tar
# check if bowties indices are unpacked on out_dir
chk_bowtie2_index(bowtie2_index_prefix)
# bowtie2
log.info('Running bowtie2...')
if args.paired_end:
bam = bowtie2_pe(args.fastqs[0], args.fastqs[1], bowtie2_index_prefix,
args.multimapping, args.nth, args.out_dir)
else:
bam = bowtie2_se(args.fastqs[0], bowtie2_index_prefix,
args.multimapping, args.nth, args.out_dir)
log.info('Removing temporary files...')
print(temp_files)
rm_f(temp_files)
log.info('Checking if BAM file is empty...')
if not int(run_shell_cmd('samtools view -c {}'.format(bam))):
raise ValueError('BAM file is empty, no reads found.')
log.info('List all files in output directory...')
ls_l(args.out_dir)
log.info('All done.')
def main():
# read params
args = parse_arguments()
log.info('Initializing and making output directory...')
mkdir_p(args.out_dir)
# declare temp arrays
temp_files = [] # files to deleted later at the end
# if bwa index is tarball then unpack it
if args.bwa_index_prefix_or_tar.endswith('.tar'):
log.info('Unpacking bwa index tar...')
tar = args.bwa_index_prefix_or_tar
# untar
untar(tar, args.out_dir)
bwa_index_prefix = os.path.join(args.out_dir,
os.path.basename(strip_ext_tar(tar)))
temp_files.append('{}.*'.format(bwa_index_prefix))
else:
bwa_index_prefix = args.bwa_index_prefix_or_tar
# check if bowties indices are unpacked on out_dir
chk_bwa_index(bwa_index_prefix)
# bwa
log.info('Running bwa...')
if args.paired_end:
bam = bwa_pe(args.fastqs[0], args.fastqs[1], bwa_index_prefix,
args.nth, args.use_bwa_mem_for_pe, args.out_dir)
else:
bam = bwa_se(args.fastqs[0], bwa_index_prefix, args.nth, args.out_dir)
log.info('Removing temporary files...')
rm_f(temp_files)
log.info('Checking if BAM file is empty...')
if not int(run_shell_cmd('samtools view -c {}'.format(bam))):
raise ValueError('BAM file is empty, no reads found.')
log.info('List all files in output directory...')
ls_l(args.out_dir)
log.info('All done.')
def macs2(ta, chrsz, gensz, pval_thresh, smooth_win, cap_num_peak, mem_gb,
out_dir):
prefix = os.path.join(out_dir, os.path.basename(strip_ext_ta(ta)))
npeak = '{}.{}.{}.narrowPeak.gz'.format(
prefix, 'pval{}'.format(pval_thresh),
human_readable_number(cap_num_peak))
# temporary files
npeak_tmp = '{}.tmp'.format(npeak)
npeak_tmp2 = '{}.tmp2'.format(npeak)
shiftsize = -int(round(float(smooth_win) / 2.0))
temp_files = []
run_shell_cmd('macs2 callpeak '
'-t {ta} -f BED -n {prefix} -g {gensz} -p {pval_thresh} '
'--shift {shiftsize} --extsize {extsize} '
'--nomodel -B --SPMR --keep-dup all --call-summits'.format(
ta=ta,
prefix=prefix,
gensz=gensz,
pval_thresh=pval_thresh,
shiftsize=shiftsize,
extsize=smooth_win,
))
run_shell_cmd(
'LC_COLLATE=C sort -k 8gr,8gr {sort_param} "{prefix}"_peaks.narrowPeak | '
'awk \'BEGIN{{OFS="\\t"}}'
'{{$4="Peak_"NR; if ($2<0) $2=0; if ($3<0) $3=0; if ($10==-1) '
'$10=$2+int(($3-$2+1)/2.0); print $0}}\' > {npeak_tmp}'.format(
sort_param=get_gnu_sort_param(mem_gb * 1024**3, ratio=0.5),
prefix=prefix,
npeak_tmp=npeak_tmp,
))
run_shell_cmd('head -n {cap_num_peak} {npeak_tmp} > {npeak_tmp2}'.format(
cap_num_peak=cap_num_peak,
npeak_tmp=npeak_tmp,
npeak_tmp2=npeak_tmp2,
))
# clip peaks between 0-chromSize.
bed_clip(npeak_tmp2, chrsz, npeak)
rm_f([npeak_tmp, npeak_tmp2])
# remove temporary files
temp_files.append("{prefix}_*".format(prefix=prefix))
rm_f(temp_files)
return npeak
def frip_shifted(ta, peak, chrsz, fraglen, out_dir):
prefix = os.path.join(out_dir, os.path.basename(strip_ext(peak)))
frip_qc = '{}.frip.qc'.format(prefix)
half_fraglen = (fraglen + 1) / 2
if get_num_lines(peak) == 0:
val1 = 0.0
else:
# due to bedtools bug when .gz is given for -a and -b
tmp2 = gunzip(peak, 'tmp2', out_dir)
cmd = 'bedtools slop -i {} -g {} '
cmd += '-s -l {} -r {} | '
cmd += 'awk \'{{if ($2>=0 && $3>=0 && $2<=$3) print $0}}\' | '
cmd += 'bedtools intersect -nonamecheck -a stdin -b {} '
cmd += '-wa -u | wc -l'
cmd = cmd.format(ta, chrsz, -half_fraglen, half_fraglen, tmp2) # peak
val1 = run_shell_cmd(cmd)
rm_f(tmp2)
val2 = get_num_lines(ta)
write_txt(frip_qc, str(float(val1) / float(val2)))
return frip_qc
def frip(ta, peak, out_dir):
prefix = os.path.join(out_dir, os.path.basename(strip_ext(peak)))
frip_qc = '{}.frip.qc'.format(prefix)
if get_num_lines(peak) == 0:
val1 = 0.0
tmp_files = []
else:
# due to bedtools bug when .gz is given for -a and -b
tmp1 = gunzip(ta, 'tmp1', out_dir)
tmp2 = gunzip(peak, 'tmp2', out_dir)
cmd = 'bedtools intersect -nonamecheck -a {} -b {} -wa -u | wc -l'
cmd = cmd.format(
tmp1, # ta
tmp2) # peak
val1 = run_shell_cmd(cmd)
tmp_files = [tmp1, tmp2]
val2 = get_num_lines(ta)
write_txt(frip_qc, str(float(val1) / float(val2)))
rm_f(tmp_files)
return frip_qc
def bwa_pe(fastq1, fastq2, ref_index_prefix, nth, use_bwa_mem_for_pe, out_dir):
basename = os.path.basename(strip_ext_fastq(fastq1))
prefix = os.path.join(out_dir, basename)
sam = '{}.sam'.format(prefix)
badcigar = '{}.badReads'.format(prefix)
bam = '{}.bam'.format(prefix)
temp_files = []
read_len = get_read_length(fastq1)
if use_bwa_mem_for_pe and read_len >= 70:
cmd = 'bwa mem -M -t {} {} {} {} | gzip -nc > {}'
cmd = cmd.format(nth, ref_index_prefix, fastq1, fastq2, sam)
temp_files.append(sam)
else:
sai1 = bwa_aln(fastq1, ref_index_prefix, nth, out_dir)
sai2 = bwa_aln(fastq2, ref_index_prefix, nth, out_dir)
cmd = 'bwa sampe {} {} {} {} {} | gzip -nc > {}'.format(
ref_index_prefix, sai1, sai2, fastq1, fastq2, sam)
temp_files.extend([sai1, sai2, sam])
run_shell_cmd(cmd)
cmd2 = 'zcat -f {} | '
cmd2 += 'awk \'BEGIN {{FS="\\t" ; OFS="\\t"}} ! /^@/ && $6!="*" '
cmd2 += '{{ cigar=$6; gsub("[0-9]+D","",cigar); '
cmd2 += 'n = split(cigar,vals,"[A-Z]"); s = 0; '
cmd2 += 'for (i=1;i<=n;i++) s=s+vals[i]; seqlen=length($10); '
cmd2 += 'if (s!=seqlen) print $1"\\t"; }}\' | '
cmd2 += 'sort | uniq > {}'
cmd2 = cmd2.format(sam, badcigar)
run_shell_cmd(cmd2)
# Remove bad CIGAR read pairs
if get_num_lines(badcigar) > 0:
cmd3 = 'zcat -f {} | grep -v -F -f {} | '
cmd3 += 'samtools view -Su - | samtools sort - -o {} -T {}'
cmd3 = cmd3.format(sam, badcigar, bam, prefix)
else:
cmd3 = 'samtools view -Su {} | samtools sort - -o {} -T {}'
cmd3 = cmd3.format(sam, bam, prefix)
run_shell_cmd(cmd3)
rm_f(temp_files)
return bam
def spr_se(ta, pseudoreplication_random_seed, out_dir):
prefix = os.path.join(out_dir, os.path.basename(strip_ext_ta(ta)))
tmp_pr1 = '{}.00'.format(prefix)
tmp_pr2 = '{}.01'.format(prefix)
ta_pr1 = '{}.pr1.tagAlign.gz'.format(prefix)
ta_pr2 = '{}.pr2.tagAlign.gz'.format(prefix)
nlines = int((get_num_lines(ta) + 1) / 2)
if pseudoreplication_random_seed == 0:
random_seed = run_shell_cmd('zcat -f {ta} | wc -c'.format(ta=ta))
log.info(
'Using input file\'s size {random_seed} as random seed for pseudoreplication.'
.format(random_seed=random_seed, ))
else:
random_seed = pseudoreplication_random_seed
log.info(
'Using a fixed integer {random_seed} as random seed for pseudoreplication.'
.format(random_seed=random_seed, ))
# bash-only
run_shell_cmd('zcat {ta} | shuf --random-source=<(openssl enc '
'-aes-256-ctr -pass pass:{random_seed} '
'-nosalt </dev/zero 2>/dev/null) | '
'split -d -l {nlines} - {prefix}.'.format(
ta=ta,
random_seed=random_seed,
nlines=nlines,
prefix=prefix,
))
run_shell_cmd('gzip -nc {tmp_pr1} > {ta_pr1}'.format(tmp_pr1=tmp_pr1,
ta_pr1=ta_pr1))
run_shell_cmd('gzip -nc {tmp_pr2} > {ta_pr2}'.format(tmp_pr2=tmp_pr2,
ta_pr2=ta_pr2))
rm_f([tmp_pr1, tmp_pr2])
return ta_pr1, ta_pr2
def xcor(ta,
speak,
mito_chr_name,
nth,
out_dir,
chip_seq_type=None,
exclusion_range_min=None,
exclusion_range_max=None):
prefix = os.path.join(out_dir, os.path.basename(strip_ext_ta(ta)))
xcor_plot_pdf = '{}.cc.plot.pdf'.format(prefix)
xcor_score = '{}.cc.qc'.format(prefix)
fraglen_txt = '{}.cc.fraglen.txt'.format(prefix)
if chip_seq_type is not None and exclusion_range_min is not None:
if exclusion_range_max is None:
exclusion_range_max = get_exclusion_range_max(ta, chip_seq_type)
exclusion_range_param = ' -x={}:{}'.format(exclusion_range_min,
exclusion_range_max)
else:
exclusion_range_param = ''
cmd1 = 'Rscript --max-ppsize=500000 $(which run_spp.R) -rf -c={} -p={} '
cmd1 += '-filtchr="{}" -savp={} -out={} {}'
cmd1 += exclusion_range_param
cmd1 = cmd1.format(ta, nth, mito_chr_name, xcor_plot_pdf, xcor_score,
'-speak={}'.format(speak) if speak >= 0 else '')
run_shell_cmd(cmd1)
cmd2 = 'sed -r \'s/,[^\\t]+//g\' -i {}'
cmd2 = cmd2.format(xcor_score)
run_shell_cmd(cmd2)
# parse xcor_score and write fraglen (3rd column) to file
cmd3 = 'echo {} > {}'.format(
parse_xcor_score(xcor_score)['estimated_fragment_len'], fraglen_txt)
run_shell_cmd(cmd3)
xcor_plot_png = pdf2png(xcor_plot_pdf, out_dir)
return xcor_plot_pdf, xcor_plot_png, xcor_score, fraglen_txt
def macs2(ta, chrsz, gensz, pval_thresh, smooth_win, cap_num_peak, out_dir):
prefix = os.path.join(out_dir, os.path.basename(strip_ext_ta(ta)))
npeak = '{}.{}.{}.narrowPeak.gz'.format(
prefix, 'pval{}'.format(pval_thresh),
human_readable_number(cap_num_peak))
# temporary files
npeak_tmp = '{}.tmp'.format(npeak)
npeak_tmp2 = '{}.tmp2'.format(npeak)
shiftsize = -int(round(float(smooth_win) / 2.0))
temp_files = []
cmd0 = 'macs2 callpeak '
cmd0 += '-t {} -f BED -n {} -g {} -p {} '
cmd0 += '--shift {} --extsize {} '
cmd0 += '--nomodel -B --SPMR '
cmd0 += '--keep-dup all --call-summits '
cmd0 = cmd0.format(ta, prefix, gensz, pval_thresh, shiftsize, smooth_win)
run_shell_cmd(cmd0)
cmd1 = 'LC_COLLATE=C sort -k 8gr,8gr "{}"_peaks.narrowPeak | '
cmd1 += 'awk \'BEGIN{{OFS="\\t"}}'
cmd1 += '{{$4="Peak_"NR; if ($2<0) $2=0; if ($3<0) $3=0; if ($10==-1) '
cmd1 += '$10=$2+int(($3-$2+1)/2.0); print $0}}\' > {}'
cmd1 = cmd1.format(prefix, npeak_tmp)
run_shell_cmd(cmd1)
cmd2 = 'head -n {} {} > {}'.format(cap_num_peak, npeak_tmp, npeak_tmp2)
run_shell_cmd(cmd2)
# clip peaks between 0-chromSize.
bed_clip(npeak_tmp2, chrsz, npeak)
rm_f([npeak_tmp, npeak_tmp2])
# remove temporary files
temp_files.append("{}_*".format(prefix))
rm_f(temp_files)
return npeak
def spr_pe(ta, out_dir):
prefix = os.path.join(out_dir, os.path.basename(strip_ext_ta(ta)))
tmp_pr1 = '{}.00'.format(prefix)
tmp_pr2 = '{}.01'.format(prefix)
ta_pr1 = '{}.pr1.tagAlign.gz'.format(prefix)
ta_pr2 = '{}.pr2.tagAlign.gz'.format(prefix)
nlines = int((get_num_lines(ta) / 2 + 1) / 2)
# bash-only
cmd1 = 'zcat -f {} | sed \'N;s/\\n/\\t/\' | '
cmd1 += 'shuf --random-source=<(openssl enc -aes-256-ctr '
cmd1 += '-pass pass:$(zcat -f {} | wc -c) '
cmd1 += '-nosalt </dev/zero 2>/dev/null) | '
cmd1 += 'split -d -l {} - {}.'
cmd1 = cmd1.format(ta, ta, nlines, prefix)
run_shell_cmd(cmd1)
cmd2 = 'zcat -f {} | '
cmd2 += 'awk \'BEGIN{{OFS="\\t"}} '
cmd2 += '{{printf "%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n'
cmd2 += '%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n",'
cmd2 += '$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12}}\' | '
cmd2 += 'gzip -nc > {}'
cmd2 = cmd2.format(tmp_pr1, ta_pr1)
run_shell_cmd(cmd2)
cmd3 = 'zcat -f {} | '
cmd3 += 'awk \'BEGIN{{OFS="\\t"}} '
cmd3 += '{{printf "%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n'
cmd3 += '%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n",'
cmd3 += '$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12}}\' | '
cmd3 += 'gzip -nc > {}'
cmd3 = cmd3.format(tmp_pr2, ta_pr2)
run_shell_cmd(cmd3)
rm_f([tmp_pr1, tmp_pr2])
return ta_pr1, ta_pr2
def bam_to_pbam(bam, ref_fa, out_dir='.'):
'''Convert BAM into pBAM.
Requirements:
- Python package `ptools_bin`
- `samtools`
'''
prefix = os.path.join(out_dir, os.path.basename(strip_ext_bam(bam)))
pbam_tmp = '{}.sorted.p.bam'.format(prefix)
pbam = '{}.p.bam'.format(prefix)
temp_files = []
if ref_fa.endswith('.gz'):
gunzipped_ref_fa = '{}.fasta'.format(
os.path.join(out_dir, os.path.basename(strip_ext_gz(ref_fa))))
run_shell_cmd('zcat -f {ref_fa} > {gunzipped_ref_fa}'.format(
ref_fa=ref_fa,
gunzipped_ref_fa=gunzipped_ref_fa,
))
temp_files.append(gunzipped_ref_fa)
else:
gunzipped_ref_fa = ref_fa
run_shell_cmd('makepBAM_genome.sh {bam} {gunzipped_ref_fa}'.format(
bam=bam,
gunzipped_ref_fa=gunzipped_ref_fa,
))
run_shell_cmd('mv {pbam_tmp} {pbam}'.format(
pbam_tmp=pbam_tmp,
pbam=pbam,
))
rm_f(temp_files)
return pbam
def peak_to_bigbed(peak, peak_type, chrsz, out_dir):
prefix = os.path.join(out_dir, os.path.basename(strip_ext(peak)))
bigbed = '{}.{}.bb'.format(prefix, peak_type)
as_file = '{}.as'.format(prefix)
chrsz_tmp = '{}.chrsz.tmp'.format(prefix)
bigbed_tmp = '{}.bb.tmp'.format(prefix)
bigbed_tmp2 = '{}.bb.tmp2'.format(prefix)
if peak_type.lower() == 'narrowpeak' or peak_type.lower() == 'regionpeak':
as_file_contents = '''table narrowPeak
"BED6+4 Peaks of signal enrichment based on pooled, normalized (interpreted) data."
(
string chrom; "Reference sequence chromosome or scaffold"
uint chromStart; "Start position in chromosome"
uint chromEnd; "End position in chromosome"
string name; "Name given to a region (preferably unique). Use . if no name is assigned"
uint score; "Indicates how dark the peak will be displayed in the browser (0-1000) "
char[1] strand; "+ or - or . for unknown"
float signalValue; "Measurement of average enrichment for the region"
float pValue; "Statistical significance of signal value (-log10). Set to -1 if not used."
float qValue; "Statistical significance with multiple-test correction applied (FDR -log10). Set to -1 if not used."
int peak; "Point-source called for this peak; 0-based offset from chromStart. Set to -1 if no point-source called."
)
'''
bed_param = '-type=bed6+4 -as={}'.format(as_file)
elif peak_type.lower() == 'broadpeak':
as_file_contents = '''table broadPeak
"BED6+3 Peaks of signal enrichment based on pooled, normalized (interpreted) data."
(
string chrom; "Reference sequence chromosome or scaffold"
uint chromStart; "Start position in chromosome"
uint chromEnd; "End position in chromosome"
string name; "Name given to a region (preferably unique). Use . if no name is assigned."
uint score; "Indicates how dark the peak will be displayed in the browser (0-1000)"
char[1] strand; "+ or - or . for unknown"
float signalValue; "Measurement of average enrichment for the region"
float pValue; "Statistical significance of signal value (-log10). Set to -1 if not used."
float qValue; "Statistical significance with multiple-test correction applied (FDR -log10). Set to -1 if not used."
)
'''
bed_param = '-type=bed6+3 -as={}'.format(as_file)
elif peak_type.lower() == 'gappedpeak':
as_file_contents = '''table gappedPeak
"This format is used to provide called regions of signal enrichment based on pooled, normalized (interpreted) data where the regions may be spliced or incorporate gaps in the genomic sequence. It is a BED12+3 format."
(
string chrom; "Reference sequence chromosome or scaffold"
uint chromStart; "Pseudogene alignment start position"
uint chromEnd; "Pseudogene alignment end position"
string name; "Name of pseudogene"
uint score; "Score of pseudogene with gene (0-1000)"
char[1] strand; "+ or - or . for unknown"
uint thickStart; "Start of where display should be thick (start codon)"
uint thickEnd; "End of where display should be thick (stop codon)"
uint reserved; "Always zero for now"
int blockCount; "Number of blocks"
int[blockCount] blockSizes; "Comma separated list of block sizes"
int[blockCount] chromStarts; "Start positions relative to chromStart"
float signalValue; "Measurement of average enrichment for the region"
float pValue; "Statistical significance of signal value (-log10). Set to -1 if not used."
float qValue; "Statistical significance with multiple-test correction applied (FDR). Set to -1 if not used."
)
'''
bed_param = '-type=bed12+3 -as={}'.format(as_file)
else:
raise Exception('Unsupported peak file type {}!'.format(peak_type))
# create temporary .as file
with open(as_file, 'w') as fp:
fp.write(as_file_contents)
cmd1 = "cat {} > {}".format(chrsz, chrsz_tmp)
run_shell_cmd(cmd1)
cmd2 = "zcat -f {} | LC_COLLATE=C sort -k1,1 -k2,2n | "
cmd2 += 'awk \'BEGIN{{OFS="\\t"}} {{if ($5>1000) $5=1000; '
cmd2 += 'if ($5<0) $5=0; print $0}}\' > {}'
cmd2 = cmd2.format(peak, bigbed_tmp)
run_shell_cmd(cmd2)
cmd3 = "bedClip {} {} {}".format(bigbed_tmp, chrsz_tmp, bigbed_tmp2)
run_shell_cmd(cmd3)
cmd4 = "bedToBigBed {} {} {} {}".format(bed_param, bigbed_tmp2, chrsz_tmp,
bigbed)
run_shell_cmd(cmd4)
# remove temporary files
rm_f([as_file, chrsz_tmp, bigbed_tmp, bigbed_tmp2])
return bigbed
def bam_is_empty(bam, nth=1):
cmd = 'samtools view -c {bam} {res_param}'.format(
bam=bam,
res_param=get_samtools_res_param('view', nth=nth),
)
return int(run_shell_cmd(cmd)) == 0
def peak_to_hammock(peak, out_dir):
peak_type = get_peak_type(peak)
prefix = os.path.join(out_dir, os.path.basename(strip_ext_peak(peak)))
hammock = '{}.{}.hammock'.format(prefix, peak_type)
hammock_tmp = '{}.tmp'.format(hammock)
hammock_tmp2 = '{}.tmp2'.format(hammock)
hammock_gz = '{}.gz'.format(hammock)
hammock_gz_tbi = '{}.gz.tbi'.format(hammock)
if get_num_lines(peak) == 0:
cmd = 'zcat -f {} | gzip -nc > {}'.format(peak, hammock_gz)
run_shell_cmd(cmd)
cmd2 = 'touch {}'.format(hammock_gz_tbi)
else:
cmd = "zcat -f {} | "
cmd += "LC_COLLATE=C sort -k1,1V -k2,2n > {}"
cmd = cmd.format(peak, hammock_tmp)
run_shell_cmd(cmd)
with open(hammock_tmp, 'r') as fin, open(hammock_tmp2, 'w') as fout:
id = 1
for line in fin:
lst = line.rstrip().split('\t')
if peak_type == 'narrowPeak' or peak_type == 'regionPeak':
fout.write(
'{0[0]}\t{0[1]}\t{0[2]}\tscorelst:[{0[6]},{0[7]},'
'{0[8]}],id:{1},'.format(lst, id))
if len(lst[3]) > 1:
fout.write('name:"' + lst[3] + '",')
if lst[5] != '.':
fout.write('strand:"' + lst[5] + '",')
if lst[9] != '-1':
fout.write('sbstroke:[' + lst[9] + ']')
elif peak_type == 'gappedPeak':
fout.write(
'{0[0]}\t{0[1]}\t{0[2]}\tscorelst:[{0[12]},{0[13]},'
'{0[14]}],id:{1},struct:{{thin:[[{0[1]},{0[2]}]],'
'thick:['.format(lst, id))
a = int(lst[1])
sizes = lst[10].split(',')
starts = lst[11].split(',')
for i in range(len(sizes)):
fout.write('[{0},{1}],'.format(
a + int(starts[i]),
a + int(starts[i]) + int(sizes[i])))
fout.write(']},')
if len(lst[3]) > 1:
fout.write('name:"' + lst[3] + '",')
if lst[5] != '.':
fout.write('strand:"' + lst[5] + '",')
elif peak_type == 'broadPeak':
fout.write(
'{0[0]}\t{0[1]}\t{0[2]}\tscorelst:[{0[6]},{0[7]}],'
'id:{1},'.format(lst, id))
if len(lst[3]) > 1:
fout.write('name:"' + lst[3] + '",')
if lst[5] != '.':
fout.write('strand:"' + lst[5] + '",')
else:
raise Exception("Unsupported peak_type {}".format(peak))
id += 1
fout.write('\n')
cmd2 = 'zcat -f {} | sort -k1,1 -k2,2n | bgzip -cf > {}'
cmd2 = cmd2.format(hammock_tmp2, hammock_gz)
run_shell_cmd(cmd2)
cmd3 = 'tabix -f -p bed {}'.format(hammock_gz)
run_shell_cmd(cmd3)
rm_f([hammock, hammock_tmp, hammock_tmp2])
return (hammock_gz, hammock_gz_tbi)
def rm_unmapped_lowq_reads_pe(bam, multimapping, mapq_thresh, nth, mem_gb,
out_dir):
"""There are pipes with multiple samtools commands.
For such pipes, use multiple threads (-@) for only one of them.
Priority is on sort > index > fixmate > view.
"""
prefix = os.path.join(out_dir, os.path.basename(strip_ext_bam(bam)))
filt_bam = '{}.filt.bam'.format(prefix)
tmp_filt_bam = '{}.tmp_filt.bam'.format(prefix)
fixmate_bam = '{}.fixmate.bam'.format(prefix)
if multimapping:
run_shell_cmd(
'samtools view -F 524 -f 2 -u {bam} | '
'samtools sort -n /dev/stdin -o {tmp_filt_bam} -T {prefix} {res_param} '
.format(
bam=bam,
tmp_filt_bam=tmp_filt_bam,
prefix=prefix,
res_param=get_samtools_res_param('view', nth=nth),
))
run_shell_cmd(
'samtools view -h {tmp_filt_bam} | '
'$(which assign_multimappers.py) -k {multimapping} --paired-end | '
'samtools fixmate -r /dev/stdin {fixmate_bam} {res_param}'.format(
tmp_filt_bam=tmp_filt_bam,
multimapping=multimapping,
fixmate_bam=fixmate_bam,
res_param=get_samtools_res_param('fixmate', nth=nth),
))
else:
run_shell_cmd(
'samtools view -F 1804 -f 2 -q {mapq_thresh} -u {bam} | '
'samtools sort -n /dev/stdin -o {tmp_filt_bam} -T {prefix} {res_param}'
.format(
mapq_thresh=mapq_thresh,
bam=bam,
tmp_filt_bam=tmp_filt_bam,
prefix=prefix,
res_param=get_samtools_res_param('sort',
nth=nth,
mem_gb=mem_gb),
))
run_shell_cmd(
'samtools fixmate -r {tmp_filt_bam} {fixmate_bam} {res_param}'.
format(
tmp_filt_bam=tmp_filt_bam,
fixmate_bam=fixmate_bam,
res_param=get_samtools_res_param('fixmate', nth=nth),
))
rm_f(tmp_filt_bam)
run_shell_cmd(
'samtools view -F 1804 -f 2 -u {fixmate_bam} | '
'samtools sort /dev/stdin -o {filt_bam} -T {prefix} {res_param}'.
format(
fixmate_bam=fixmate_bam,
filt_bam=filt_bam,
prefix=prefix,
res_param=get_samtools_res_param('sort', nth=nth, mem_gb=mem_gb),
))
rm_f(fixmate_bam)
log.info('Checking if filtered (but not deduped) BAM is empty '
'after filtering with "samtools view -F 1804 -f 2".')
if bam_is_empty(filt_bam, nth):
raise ValueError(
'No reads found aftering filtering "samtools fixmate"d PE BAM with '
'"samtools view -F 1804 -f 2". '
'Reads are not properly paired even though mapping rate is good? ')
return filt_bam
def macs2_signal_track(ta, ctl_ta, chrsz, gensz, pval_thresh, shift, fraglen,
ctl_subsample, ctl_paired_end, mem_gb, out_dir):
basename_ta = os.path.basename(strip_ext_ta(ta))
if ctl_ta:
if ctl_subsample:
if ctl_paired_end:
ctl_ta = subsample_ta_pe(ctl_ta,
ctl_subsample,
non_mito=False,
mito_chr_name=None,
r1_only=False,
out_dir=out_dir)
else:
ctl_ta = subsample_ta_se(ctl_ta,
ctl_subsample,
non_mito=False,
mito_chr_name=None,
out_dir=out_dir)
basename_ctl_ta = os.path.basename(strip_ext_ta(ctl_ta))
basename_prefix = '{}_x_{}'.format(basename_ta, basename_ctl_ta)
if len(basename_prefix) > 200: # UNIX cannot have len(filename) > 255
basename_prefix = '{}_x_control'.format(basename_ta)
else:
basename_prefix = basename_ta
prefix = os.path.join(out_dir, basename_prefix)
fc_bigwig = '{}.fc.signal.bigwig'.format(prefix)
pval_bigwig = '{}.pval.signal.bigwig'.format(prefix)
# temporary files
fc_bedgraph = '{}.fc.signal.bedgraph'.format(prefix)
fc_bedgraph_srt = '{}.fc.signal.srt.bedgraph'.format(prefix)
pval_bedgraph = '{}.pval.signal.bedgraph'.format(prefix)
pval_bedgraph_srt = '{}.pval.signal.srt.bedgraph'.format(prefix)
temp_files = []
run_shell_cmd(
' macs2 callpeak '
'-t {ta} {ctl_param} -f BED -n {prefix} -g {gensz} -p {pval_thresh} '
'--nomodel --shift {shiftsize} --extsize {extsize} --keep-dup all -B --SPMR'
.format(
ta=ta,
ctl_param='-c {ctl_ta}'.format(ctl_ta=ctl_ta) if ctl_ta else '',
prefix=prefix,
gensz=gensz,
pval_thresh=pval_thresh,
shiftsize=0,
extsize=fraglen,
))
run_shell_cmd('macs2 bdgcmp -t "{prefix}_treat_pileup.bdg" '
'-c "{prefix}_control_lambda.bdg" '
'--o-prefix "{prefix}" -m FE '.format(prefix=prefix, ))
run_shell_cmd('bedtools slop -i "{prefix}_FE.bdg" -g {chrsz} -b 0 | '
'awk \'{{if ($3 != -1) print $0}}\' |'
'bedClip stdin {chrsz} {fc_bedgraph}'.format(
prefix=prefix,
chrsz=chrsz,
fc_bedgraph=fc_bedgraph,
))
# sort and remove any overlapping regions in bedgraph by comparing two lines in a row
run_shell_cmd(
'LC_COLLATE=C sort -k1,1 -k2,2n {sort_param} {fc_bedgraph} | '
'awk \'BEGIN{{OFS="\\t"}}{{if (NR==1 || NR>1 && (prev_chr!=$1 || '
'prev_chr==$1 && prev_chr_e<=$2)) '
'{{print $0}}; prev_chr=$1; prev_chr_e=$3;}}\' > {fc_bedgraph_srt}'.
format(sort_param=get_gnu_sort_param(mem_gb * 1024**3, ratio=0.5),
fc_bedgraph=fc_bedgraph,
fc_bedgraph_srt=fc_bedgraph_srt))
rm_f(fc_bedgraph)
run_shell_cmd(
'bedGraphToBigWig {fc_bedgraph_srt} {chrsz} {fc_bigwig}'.format(
fc_bedgraph_srt=fc_bedgraph_srt,
chrsz=chrsz,
fc_bigwig=fc_bigwig,
))
rm_f(fc_bedgraph_srt)
# sval counts the number of tags per million in the (compressed) BED file
sval = float(get_num_lines(ta)) / 1000000.0
run_shell_cmd('macs2 bdgcmp -t "{prefix}_treat_pileup.bdg" '
'-c "{prefix}_control_lambda.bdg" '
'--o-prefix {prefix} -m ppois -S {sval}'.format(
prefix=prefix,
sval=sval,
))
run_shell_cmd('bedtools slop -i "{prefix}_ppois.bdg" -g {chrsz} -b 0 | '
'awk \'{{if ($3 != -1) print $0}}\' |'
'bedClip stdin {chrsz} {pval_bedgraph}'.format(
prefix=prefix,
chrsz=chrsz,
pval_bedgraph=pval_bedgraph,
))
# sort and remove any overlapping regions in bedgraph by comparing two lines in a row
run_shell_cmd(
'LC_COLLATE=C sort -k1,1 -k2,2n {sort_param} {pval_bedgraph} | '
'awk \'BEGIN{{OFS="\\t"}}{{if (NR==1 || NR>1 && (prev_chr!=$1 || '
'prev_chr==$1 && prev_chr_e<=$2)) '
'{{print $0}}; prev_chr=$1; prev_chr_e=$3;}}\' > {pval_bedgraph_srt}'.
format(
sort_param=get_gnu_sort_param(mem_gb * 1024**3, ratio=0.5),
pval_bedgraph=pval_bedgraph,
pval_bedgraph_srt=pval_bedgraph_srt,
))
rm_f(pval_bedgraph)
run_shell_cmd(
'bedGraphToBigWig {pval_bedgraph_srt} {chrsz} {pval_bigwig}'.format(
pval_bedgraph_srt=pval_bedgraph_srt,
chrsz=chrsz,
pval_bigwig=pval_bigwig))
rm_f(pval_bedgraph_srt)
# remove temporary files
temp_files.append("{prefix}_*".format(prefix=prefix))
rm_f(temp_files)
return fc_bigwig, pval_bigwig
def macs2_signal_track(ta, chrsz, gensz, pval_thresh, smooth_win, out_dir):
prefix = os.path.join(out_dir, os.path.basename(strip_ext_ta(ta)))
fc_bigwig = '{}.fc.signal.bigwig'.format(prefix)
pval_bigwig = '{}.pval.signal.bigwig'.format(prefix)
# temporary files
fc_bedgraph = '{}.fc.signal.bedgraph'.format(prefix)
fc_bedgraph_srt = '{}.fc.signal.srt.bedgraph'.format(prefix)
pval_bedgraph = '{}.pval.signal.bedgraph'.format(prefix)
pval_bedgraph_srt = '{}.pval.signal.srt.bedgraph'.format(prefix)
shiftsize = -int(round(float(smooth_win) / 2.0))
temp_files = []
cmd0 = 'macs2 callpeak '
cmd0 += '-t {} -f BED -n {} -g {} -p {} '
cmd0 += '--shift {} --extsize {} '
cmd0 += '--nomodel -B --SPMR '
cmd0 += '--keep-dup all --call-summits '
cmd0 = cmd0.format(ta, prefix, gensz, pval_thresh, shiftsize, smooth_win)
run_shell_cmd(cmd0)
cmd3 = 'macs2 bdgcmp -t "{}"_treat_pileup.bdg '
cmd3 += '-c "{}"_control_lambda.bdg '
cmd3 += '--o-prefix "{}" -m FE '
cmd3 = cmd3.format(prefix, prefix, prefix)
run_shell_cmd(cmd3)
cmd4 = 'bedtools slop -i "{}"_FE.bdg -g {} -b 0 | '
cmd4 += 'bedClip stdin {} {}'
cmd4 = cmd4.format(prefix, chrsz, chrsz, fc_bedgraph)
run_shell_cmd(cmd4)
# sort and remove any overlapping regions in bedgraph by comparing two lines in a row
cmd5 = 'LC_COLLATE=C sort -k1,1 -k2,2n {} | ' \
'awk \'BEGIN{{OFS="\\t"}}{{if (NR==1 || NR>1 && (prev_chr!=$1 '\
'|| prev_chr==$1 && prev_chr_e<=$2)) ' \
'{{print $0}}; prev_chr=$1; prev_chr_e=$3;}}\' > {}'.format(
fc_bedgraph,
fc_bedgraph_srt)
run_shell_cmd(cmd5)
rm_f(fc_bedgraph)
cmd6 = 'bedGraphToBigWig {} {} {}'
cmd6 = cmd6.format(fc_bedgraph_srt, chrsz, fc_bigwig)
run_shell_cmd(cmd6)
rm_f(fc_bedgraph_srt)
# sval counts the number of tags per million in the (compressed) BED file
sval = float(get_num_lines(ta)) / 1000000.0
cmd7 = 'macs2 bdgcmp -t "{}"_treat_pileup.bdg '
cmd7 += '-c "{}"_control_lambda.bdg '
cmd7 += '--o-prefix {} -m ppois -S {}'
cmd7 = cmd7.format(prefix, prefix, prefix, sval)
run_shell_cmd(cmd7)
cmd8 = 'bedtools slop -i "{}"_ppois.bdg -g {} -b 0 | '
cmd8 += 'bedClip stdin {} {}'
cmd8 = cmd8.format(prefix, chrsz, chrsz, pval_bedgraph)
run_shell_cmd(cmd8)
# sort and remove any overlapping regions in bedgraph by comparing two lines in a row
cmd9 = 'LC_COLLATE=C sort -k1,1 -k2,2n {} | ' \
'awk \'BEGIN{{OFS="\\t"}}{{if (NR==1 || NR>1 && (prev_chr!=$1 '\
'|| prev_chr==$1 && prev_chr_e<=$2)) ' \
'{{print $0}}; prev_chr=$1; prev_chr_e=$3;}}\' > {}'.format(
pval_bedgraph,
pval_bedgraph_srt)
run_shell_cmd(cmd9)
rm_f(pval_bedgraph)
cmd10 = 'bedGraphToBigWig {} {} {}'
cmd10 = cmd10.format(pval_bedgraph_srt, chrsz, pval_bigwig)
run_shell_cmd(cmd10)
rm_f(pval_bedgraph_srt)
# remove temporary files
temp_files.append("{}_*".format(prefix))
rm_f(temp_files)
return fc_bigwig, pval_bigwig
def macs2_signal_track(ta, ctl_ta, chrsz, gensz, pval_thresh, shift, fraglen,
ctl_subsample, ctl_paired_end, out_dir):
basename_ta = os.path.basename(strip_ext_ta(ta))
if ctl_ta:
if ctl_subsample:
if ctl_paired_end:
ctl_ta = subsample_ta_pe(ctl_ta,
ctl_subsample,
non_mito=False,
mito_chr_name=None,
r1_only=False,
out_dir=out_dir)
else:
ctl_ta = subsample_ta_se(ctl_ta,
ctl_subsample,
non_mito=False,
mito_chr_name=None,
out_dir=out_dir)
basename_ctl_ta = os.path.basename(strip_ext_ta(ctl_ta))
basename_prefix = '{}_x_{}'.format(basename_ta, basename_ctl_ta)
if len(basename_prefix) > 200: # UNIX cannot have len(filename) > 255
basename_prefix = '{}_x_control'.format(basename_ta)
else:
basename_prefix = basename_ta
prefix = os.path.join(out_dir, basename_prefix)
fc_bigwig = '{}.fc.signal.bigwig'.format(prefix)
pval_bigwig = '{}.pval.signal.bigwig'.format(prefix)
# temporary files
fc_bedgraph = '{}.fc.signal.bedgraph'.format(prefix)
fc_bedgraph_srt = '{}.fc.signal.srt.bedgraph'.format(prefix)
pval_bedgraph = '{}.pval.signal.bedgraph'.format(prefix)
pval_bedgraph_srt = '{}.pval.signal.srt.bedgraph'.format(prefix)
temp_files = []
cmd0 = ' macs2 callpeak '
cmd0 += '-t {} {} -f BED -n {} -g {} -p {} '
cmd0 += '--nomodel --shift {} --extsize {} --keep-dup all -B --SPMR'
cmd0 = cmd0.format(ta, '-c {}'.format(ctl_ta) if ctl_ta else '', prefix,
gensz, pval_thresh, 0, fraglen)
run_shell_cmd(cmd0)
cmd3 = 'macs2 bdgcmp -t "{}"_treat_pileup.bdg '
cmd3 += '-c "{}"_control_lambda.bdg '
cmd3 += '--o-prefix "{}" -m FE '
cmd3 = cmd3.format(prefix, prefix, prefix)
run_shell_cmd(cmd3)
cmd4 = 'bedtools slop -i "{}"_FE.bdg -g {} -b 0 | '
cmd4 += 'awk \'{{if ($3 != -1) print $0}}\' |'
cmd4 += 'bedClip stdin {} {}'
cmd4 = cmd4.format(prefix, chrsz, chrsz, fc_bedgraph)
run_shell_cmd(cmd4)
# sort and remove any overlapping regions in bedgraph by comparing two lines in a row
cmd5 = 'LC_COLLATE=C sort -k1,1 -k2,2n {} | ' \
'awk \'BEGIN{{OFS="\\t"}}{{if (NR==1 || NR>1 && (prev_chr!=$1 || '\
'prev_chr==$1 && prev_chr_e<=$2)) ' \
'{{print $0}}; prev_chr=$1; prev_chr_e=$3;}}\' > {}'.format(
fc_bedgraph,
fc_bedgraph_srt)
run_shell_cmd(cmd5)
rm_f(fc_bedgraph)
cmd6 = 'bedGraphToBigWig {} {} {}'
cmd6 = cmd6.format(fc_bedgraph_srt, chrsz, fc_bigwig)
run_shell_cmd(cmd6)
rm_f(fc_bedgraph_srt)
# sval counts the number of tags per million in the (compressed) BED file
sval = float(get_num_lines(ta)) / 1000000.0
cmd7 = 'macs2 bdgcmp -t "{}"_treat_pileup.bdg '
cmd7 += '-c "{}"_control_lambda.bdg '
cmd7 += '--o-prefix {} -m ppois -S {}'
cmd7 = cmd7.format(prefix, prefix, prefix, sval)
run_shell_cmd(cmd7)
cmd8 = 'bedtools slop -i "{}"_ppois.bdg -g {} -b 0 | '
cmd8 += 'awk \'{{if ($3 != -1) print $0}}\' |'
cmd8 += 'bedClip stdin {} {}'
cmd8 = cmd8.format(prefix, chrsz, chrsz, pval_bedgraph)
run_shell_cmd(cmd8)
# sort and remove any overlapping regions in bedgraph by comparing two lines in a row
cmd9 = 'LC_COLLATE=C sort -k1,1 -k2,2n {} | ' \
'awk \'BEGIN{{OFS="\\t"}}{{if (NR==1 || NR>1 && (prev_chr!=$1 || '\
'prev_chr==$1 && prev_chr_e<=$2)) ' \
'{{print $0}}; prev_chr=$1; prev_chr_e=$3;}}\' > {}'.format(
pval_bedgraph,
pval_bedgraph_srt)
run_shell_cmd(cmd9)
rm_f(pval_bedgraph)
cmd10 = 'bedGraphToBigWig {} {} {}'
cmd10 = cmd10.format(pval_bedgraph_srt, chrsz, pval_bigwig)
run_shell_cmd(cmd10)
rm_f(pval_bedgraph_srt)
# remove temporary files
temp_files.append("{}_*".format(prefix))
rm_f(temp_files)
return fc_bigwig, pval_bigwig
def macs2(ta, ctl_ta, chrsz, gensz, pval_thresh, shift, fraglen, cap_num_peak,
ctl_subsample, ctl_paired_end, mem_gb, out_dir):
basename_ta = os.path.basename(strip_ext_ta(ta))
if ctl_ta:
if ctl_subsample:
if ctl_paired_end:
ctl_ta = subsample_ta_pe(ctl_ta,
ctl_subsample,
non_mito=False,
mito_chr_name=None,
r1_only=False,
out_dir=out_dir)
else:
ctl_ta = subsample_ta_se(ctl_ta,
ctl_subsample,
non_mito=False,
mito_chr_name=None,
out_dir=out_dir)
basename_ctl_ta = os.path.basename(strip_ext_ta(ctl_ta))
basename_prefix = '{}_x_{}'.format(basename_ta, basename_ctl_ta)
if len(basename_prefix) > 200: # UNIX cannot have len(filename) > 255
basename_prefix = '{}_x_control'.format(basename_ta)
else:
basename_prefix = basename_ta
prefix = os.path.join(out_dir, basename_prefix)
npeak = '{}.{}.{}.narrowPeak.gz'.format(
prefix, 'pval{}'.format(pval_thresh),
human_readable_number(cap_num_peak))
npeak_tmp = '{}.tmp'.format(npeak)
npeak_tmp2 = '{}.tmp2'.format(npeak)
temp_files = []
run_shell_cmd(
' macs2 callpeak '
'-t {ta} {ctl_param} -f BED -n {prefix} -g {gensz} -p {pval_thresh} '
'--nomodel --shift {shiftsize} --extsize {extsize} --keep-dup all -B --SPMR'
.format(
ta=ta,
ctl_param='-c {ctl_ta}'.format(ctl_ta=ctl_ta) if ctl_ta else '',
prefix=prefix,
gensz=gensz,
pval_thresh=pval_thresh,
shiftsize=0,
extsize=fraglen,
))
run_shell_cmd(
'LC_COLLATE=C sort -k 8gr,8gr {sort_param} "{prefix}_peaks.narrowPeak" | '
'awk \'BEGIN{{OFS="\\t"}}'
'{{$4="Peak_"NR; if ($2<0) $2=0; if ($3<0) $3=0; if ($10==-1) '
'$10=$2+int(($3-$2+1)/2.0); print $0}}\' > {npeak_tmp}'.format(
sort_param=get_gnu_sort_param(mem_gb * 1024**3, ratio=0.5),
prefix=prefix,
npeak_tmp=npeak_tmp,
))
run_shell_cmd('head -n {cap_num_peak} {npeak_tmp} > {npeak_tmp2}'.format(
cap_num_peak=cap_num_peak,
npeak_tmp=npeak_tmp,
npeak_tmp2=npeak_tmp2,
))
# clip peaks between 0-chromSize.
bed_clip(npeak_tmp2, chrsz, npeak)
rm_f([npeak_tmp, npeak_tmp2])
# remove temporary files
temp_files.append("{prefix}_*".format(prefix=prefix))
rm_f(temp_files)
return npeak
def bwa_pe(fastq1, fastq2, ref_index_prefix, nth, mem_gb, use_bwa_mem_for_pe,
bwa_mem_read_len_limit, rescue_reads_for_bwa_mem, out_dir):
basename = os.path.basename(strip_ext_fastq(fastq1))
prefix = os.path.join(out_dir, basename)
sam = '{}.sam'.format(prefix)
badcigar = '{}.badReads'.format(prefix)
bam = '{}.bam'.format(prefix)
temp_files = []
read_len = get_read_length(fastq1)
log.info('Guessed read length of R1 FASTQ: {read_len}'.format(
read_len=read_len, ))
if use_bwa_mem_for_pe and read_len >= bwa_mem_read_len_limit:
log.info('Use bwa mem.')
cmd = 'bwa mem -M {extra_param} -t {nth} {ref_index_prefix} {fastq1} {fastq2} | gzip -nc > {sam}'.format(
extra_param='-P' if rescue_reads_for_bwa_mem else '',
nth=nth,
ref_index_prefix=ref_index_prefix,
fastq1=fastq1,
fastq2=fastq2,
sam=sam,
)
temp_files.append(sam)
else:
log.info('Use bwa aln for each (R1 and R2) and then bwa sampe.')
sai1 = bwa_aln(fastq1, ref_index_prefix, nth, out_dir)
sai2 = bwa_aln(fastq2, ref_index_prefix, nth, out_dir)
cmd = 'bwa sampe {} {} {} {} {} | gzip -nc > {}'.format(
ref_index_prefix, sai1, sai2, fastq1, fastq2, sam)
temp_files.extend([sai1, sai2, sam])
run_shell_cmd(cmd)
cmd2 = 'zcat -f {} | '
cmd2 += 'awk \'BEGIN {{FS="\\t" ; OFS="\\t"}} ! /^@/ && $6!="*" '
cmd2 += '{{ cigar=$6; gsub("[0-9]+D","",cigar); '
cmd2 += 'n = split(cigar,vals,"[A-Z]"); s = 0; '
cmd2 += 'for (i=1;i<=n;i++) s=s+vals[i]; seqlen=length($10); '
cmd2 += 'if (s!=seqlen) print $1"\\t"; }}\' | '
cmd2 += 'sort | uniq > {}'
cmd2 = cmd2.format(sam, badcigar)
run_shell_cmd(cmd2)
# Remove bad CIGAR read pairs
if get_num_lines(badcigar) > 0:
run_shell_cmd(
'zcat -f {sam} | grep -v -F -f {badcigar} | '
'samtools view -Su /dev/stdin | samtools sort /dev/stdin -o {bam} -T {prefix} {res_param}'
.format(
sam=sam,
badcigar=badcigar,
bam=bam,
prefix=prefix,
res_param=get_samtools_res_param('sort',
nth=nth,
mem_gb=mem_gb),
))
else:
run_shell_cmd(
'samtools view -Su {sam} | samtools sort /dev/stdin -o {bam} -T {prefix} {res_param}'
.format(
sam=sam,
bam=bam,
prefix=prefix,
res_param=get_samtools_res_param('sort',
nth=nth,
mem_gb=mem_gb),
))
rm_f(temp_files)
return bam
def rm_unmapped_lowq_reads_pe(bam, multimapping, mapq_thresh, nth, mem_gb,
out_dir):
"""There are pipes with multiple samtools commands.
For such pipes, use multiple threads (-@) for only one of them.
Priority is on sort > index > fixmate > view.
"""
prefix = os.path.join(out_dir, os.path.basename(strip_ext_bam(bam)))
filt_bam = '{}.filt.bam'.format(prefix)
tmp_filt_bam = '{}.tmp_filt.bam'.format(prefix)
fixmate_bam = '{}.fixmate.bam'.format(prefix)
if multimapping:
run_shell_cmd(
'samtools view -F 524 -f 2 -u {bam} | '
'samtools sort -n /dev/stdin -o {tmp_filt_bam} -T {prefix} {res_param} '
.format(
bam=bam,
tmp_filt_bam=tmp_filt_bam,
prefix=prefix,
res_param=get_samtools_res_param('view', nth=nth),
))
run_shell_cmd(
'samtools view -h {tmp_filt_bam} | '
'$(which assign_multimappers.py) -k {multimapping} --paired-end | '
'samtools fixmate -r /dev/stdin {fixmate_bam} {res_param}'.format(
tmp_filt_bam=tmp_filt_bam,
multimapping=multimapping,
fixmate_bam=fixmate_bam,
res_param=get_samtools_res_param('fixmate', nth=nth),
))
else:
run_shell_cmd(
'samtools view -F 1804 -f 2 -q {mapq_thresh} -u {bam} | '
'samtools sort -n /dev/stdin -o {tmp_filt_bam} -T {prefix} {res_param}'
.format(
mapq_thresh=mapq_thresh,
bam=bam,
tmp_filt_bam=tmp_filt_bam,
prefix=prefix,
res_param=get_samtools_res_param('sort',
nth=nth,
mem_gb=mem_gb),
))
run_shell_cmd(
'samtools fixmate -r {tmp_filt_bam} {fixmate_bam} {res_param}'.
format(
tmp_filt_bam=tmp_filt_bam,
fixmate_bam=fixmate_bam,
res_param=get_samtools_res_param('fixmate', nth=nth),
))
rm_f(tmp_filt_bam)
run_shell_cmd(
'samtools view -F 1804 -f 2 -u {fixmate_bam} | '
'samtools sort /dev/stdin -o {filt_bam} -T {prefix} {res_param}'.
format(
fixmate_bam=fixmate_bam,
filt_bam=filt_bam,
prefix=prefix,
res_param=get_samtools_res_param('sort', nth=nth, mem_gb=mem_gb),
))
rm_f(fixmate_bam)
return filt_bam
