def run_filter(args): sz_utils.check_if_files_exist(args.ac_file) fOUT = None if args.out == sys.stdout: fOUT = sys.stdout else: sz_utils.make_dirs_if_necessary(args.out) fOUT = open(args.out, 'w') before, after = 0, 0 # number of SNPs before and after filteration with open(args.ac_file, 'r') as fAC: for line in fAC: tmp_line = line.strip().split("\t") before += 1 ref_base = tmp_line[2] alt_base = tmp_line[3] fail = 0 for i in range(len(tmp_line[4:])): fail = apply_filter(tmp_line[4:][i], fail, args) if fail: break if not fail: fOUT.write(line) after += 1 fOUT.close() ColorText().info("Number of SNPs before filtering: %d\n" % (before), "stderr") ColorText().info("Number of SNPs after filtering: %d\n" % (after), "stderr")
def run_merge(args): ''' combine allele counts across replicates ''' allele_counts = collections.defaultdict(list) data = collections.defaultdict(list) for ac_file in args.acs: sz_utils.check_if_files_exist(ac_file) ColorText().info("[poolseq_tk] reading and updating allele counts from %s ..." %(ac_file), "stderr") with open(ac_file) as fAC: for line in fAC: tmp_line = line.strip().split() pos = int(tmp_line[1]) if not pos in data: data[pos] = tmp_line[0:4] if not pos in allele_counts: allele_counts[pos] = map(int, tmp_line[4].split(':')) else: allele_counts[pos] = map(sum, zip(allele_counts[pos], map(int, tmp_line[4].split(':')))) ColorText().info(" [done]\n", "stderr") # output to file fOUT = None if args.out == sys.stdout: fOUT = sys.stdout else: sz_utils.make_dirs_if_necessary(args.out) fOUT = open(args.out, 'w') ColorText().info("[poolseq_tk] outputting to %s ..." %(fOUT.name), "stderr") for pos in sorted(allele_counts.iterkeys()): fOUT.write("%s\t%s\n" %("\t".join(data[pos]), ":".join(map(str, allele_counts[pos])))) ColorText().info(" [done]\n", "stderr")
def run_filter(args): sz_utils.check_if_files_exist(args.ac_file) fOUT = None if args.out == sys.stdout: fOUT = sys.stdout else: sz_utils.make_dirs_if_necessary(args.out) fOUT = open(args.out, 'w') before, after = 0, 0 # number of SNPs before and after filteration with open(args.ac_file, 'r') as fAC: for line in fAC: tmp_line = line.strip().split("\t") before += 1 ref_base = tmp_line[2] alt_base = tmp_line[3] fail = 0 for i in range(len(tmp_line[4:])): fail = apply_filter(tmp_line[4:][i], fail, args) if fail: break if not fail: fOUT.write(line) after += 1 fOUT.close() ColorText().info("Number of SNPs before filtering: %d\n" %(before), "stderr") ColorText().info("Number of SNPs after filtering: %d\n" %(after), "stderr")
def run_overlap(args): ''' getting SNPs identified from both pools ''' sz_utils.check_if_files_exist(args.file_a, args.file_b) snp_a = collections.defaultdict(list) with open(args.file_a, 'r') as fA: for line in fA: tmp_line = line.strip().split("\t") snp_a[int(tmp_line[1])] = tmp_line ColorText().info( "[poolseq_tk]: %d SNPs parsed from %s\n" % (len(snp_a), os.path.basename(args.file_a)), "stderr") sz_utils.make_dirs_if_necessary(args.out) num_overlapion = 0 with open(args.out, 'w') as fOUT: with open(args.file_b, 'r') as fB: for line in fB: tmp_line = line.strip().split("\t") if int(tmp_line[1]) in snp_a: num_overlapion += 1 fOUT.write("%s\t%s\n" % ("\t".join(snp_a[int( tmp_line[1])]), "\t".join(tmp_line[-4:]))) ColorText().info( "[poolseq_tk]: %d SNPs identified from both pools\n" % (num_overlapion), "stderr")
def read_mpileup(mpileup_file, offset): ''' read certain columns in a pileup file into a dictionary of tuple ''' ColorText().info("[poolseq_tk]: reading %s ..." % (mpileup_file), "stderr") mpileup_info = collections.defaultdict(tuple) chr = "" sz_utils.check_if_files_exist(mpileup_file) with open(mpileup_file, 'r') as fMPILEUP: for line in fMPILEUP: tmp_line = line.strip().split("\t") chr = tmp_line[0] ''' key: SNP position in integer value: a tuple with two elements 1) ref base at the position 2) reads bases covering that position if the coverage at that position > 0 else N/A ''' # if int(tmp_line[3]) == 0: # the forth column: coverage at a position # mpileup_info[int(tmp_line[1])+offset] = (tmp_line[2].upper(), "N/A") if int(tmp_line[3]) > 0: mpileup_info[int(tmp_line[1]) + offset] = (tmp_line[2].upper(), tmp_line[4]) ColorText().info(" [done]\n", "stderr") return chr, mpileup_info
def run_prepVCF(args): sz_utils.check_if_files_exist(args.infile) dfst = collections.defaultdict(list) if args.ifst: dfst = getFst(args.ifst, dfst) dfilters = getFilters(args.filters) sz_utils.make_dirs_if_necessary(args.out) fOUT = open(args.out, 'w') outVCFHeaders(args.samples, fOUT) with open(args.infile, "r") as fIN: for line in fIN: tmp_line = line.strip().split("\t") chr = tmp_line[0] pos = int(tmp_line[1]) refBase = tmp_line[2] altBase = tmp_line[3] pval = float(tmp_line[8]) corrPval = float(tmp_line[10]) ratio = float(tmp_line[-1]) fst = -1.0 if chr in dfst: for j in range(len(dfst[chr])): if pos == dfst[chr][j][0]: fst = float(dfst[chr][j][1]) dfst[chr].pop(j) break if "ratio" in dfilters: if ((dfilters["ratio"][0] == '<' and ratio >= dfilters["ratio"][1]) or dfilters["ratio"][0] == '>' and ratio <= dfilters["ratio"][1]): continue if "pval" in dfilters: # fix later pass if "corrPval" in dfilters: # fix later pass fOUT.write("%s\t%s\t.\t%s\t%s\t.\t.\t" % (chr, pos, refBase, altBase)) if fst == -1.0: fOUT.write("pval=%.5g;corrPval=%.5f;ratio=%.5f\t" % (pval, corrPval, ratio)) else: fOUT.write("pval=%.5g;corrPval=%.5f;ratio=%.5f;fst=%.5f\t" % (pval, corrPval, ratio, fst)) fOUT.write("GT:Table") poolIndex = 1 for i in range(len(tmp_line[4:-3])): table = tmp_line[4:-3][i].replace(':', '-') fOUT.write("\t./.:%s" % (table)) fOUT.write("\n") fOUT.close()
def run_prepVCF(args): sz_utils.check_if_files_exist(args.infile) dfst = collections.defaultdict(list) if args.ifst: dfst = getFst(args.ifst, dfst) dfilters = getFilters(args.filters) sz_utils.make_dirs_if_necessary(args.out) fOUT = open(args.out, 'w') outVCFHeaders(args.samples, fOUT) with open(args.infile, "r") as fIN: for line in fIN: tmp_line = line.strip().split("\t") chr = tmp_line[0] pos = int(tmp_line[1]) refBase = tmp_line[2] altBase = tmp_line[3] pval = float(tmp_line[8]) corrPval = float(tmp_line[10]) ratio = float(tmp_line[-1]) fst = -1.0 if chr in dfst: for j in range(len(dfst[chr])): if pos == dfst[chr][j][0]: fst = float(dfst[chr][j][1]) dfst[chr].pop(j) break if "ratio" in dfilters: if ((dfilters["ratio"][0] == '<' and ratio >= dfilters["ratio"][1]) or dfilters["ratio"][0] == '>' and ratio <= dfilters["ratio"][1]): continue if "pval" in dfilters: # fix later pass if "corrPval" in dfilters: # fix later pass fOUT.write("%s\t%s\t.\t%s\t%s\t.\t.\t" %(chr, pos, refBase, altBase)) if fst == -1.0: fOUT.write("pval=%.5g;corrPval=%.5f;ratio=%.5f\t" %(pval, corrPval, ratio)) else: fOUT.write("pval=%.5g;corrPval=%.5f;ratio=%.5f;fst=%.5f\t" %(pval, corrPval, ratio, fst)) fOUT.write("GT:Table") poolIndex = 1 for i in range(len(tmp_line[4:-3])): table = tmp_line[4:-3][i].replace(':', '-') fOUT.write("\t./.:%s" %(table)) fOUT.write("\n") fOUT.close()
def get_SNPs(snps_file): ''' read the SNP positions into a dictionary of tuple ''' dSNPs = collections.defaultdict(tuple) sz_utils.check_if_files_exist(snps_file) with open(snps_file, 'r') as fSNP: for line in fSNP: if not line.startswith('#'): tmp_line = line.strip().split("\t") pos = int(tmp_line[0]) ref_base = tmp_line[1] alt_base = tmp_line[2] if pos not in dSNPs: dSNPs[pos] = (ref_base, alt_base) return dSNPs
def run_view(args): check_if_files_exist(args.ipileup) make_dirs_if_necessary(args.out) dSNPs = getSNPs(args.isnp) fOUT = open(args.out, 'w') # nRemoved = 0 with open(args.ipileup, 'r') as fIN: for line in fIN: tmp_line = line.strip().split("\t") chr = tmp_line[0] pos = int(tmp_line[1]) if (chr, pos) in dSNPs: cov = int(tmp_line[3]) ref_base = tmp_line[2].upper() alt_base = dSNPs[chr, pos][1] if ref_base == dSNPs[chr, pos][0]: if cov > 0: reads_bases = tmp_line[4] reads_bases_parsed = parseReadsBases( ref_base, alt_base, reads_bases) fOUT.write( "%s\t%d\t%s\t%s\t%s\n" % (chr, pos, ref_base, alt_base, reads_bases_parsed)) # reads_bases_parsed, nReadsBases, nRefBases, dMultiBases, dIndels = parseReadsBases(reads_bases, ref_base, alt_base) # the following is a checkup on other alleles # at this moment this checkup is inactive # number of alleles (SNPs, Indels) other than the alternative allele # nMultiBases = sum(dMultiBases.values()) + sum(dIndels.values()) # if (nReadsBases == nRefBases or # (nMultiBases)/float(nReadsBases) <= 0.05): # out.write("%s\t%d\t%s\t%s\t%s\n" %(chr, pos, ref_base, alt_base, # reads_bases_parsed)) # else: # nRemoved += 1 # print pos, ref_base, alt_base, reads_bases, reads_bases_parsed # print dMultiBases # print dIndels # print else: sys.stderr.write("reference base not consistent\n") sys.stderr.write(line) sys.exit() del dSNPs[chr, pos] fOUT.close()
def read_mpileup(mpileup_file, offset): ''' read certain columns in a pileup file into a dictionary of tuple ''' ColorText().info("[poolseq_tk]: reading %s ..." %(mpileup_file), "stderr") dMpileups = collections.defaultdict(tuple) chr = "" sz_utils.check_if_files_exist(mpileup_file) with open(mpileup_file, 'r') as fMPILEUP: for line in fMPILEUP: tmp_line = line.strip().split("\t") chr = tmp_line[0] pos = int(tmp_line[1]) cov = int(tmp_line[3]) ref_base = tmp_line[2].upper() if cov > 0: reads_bases = tmp_line[4] dMpileups[pos+offset] = (ref_base, reads_bases) ColorText().info(" [done]\n", "stderr") return chr, dMpileups
def run_view(args): check_if_files_exist(args.ipileup) make_dirs_if_necessary(args.out) dSNPs = getSNPs(args.isnp) fOUT = open(args.out, 'w') # nRemoved = 0 with open(args.ipileup, 'r') as fIN: for line in fIN: tmp_line = line.strip().split("\t") chr = tmp_line[0] pos = int(tmp_line[1]) if (chr, pos) in dSNPs: cov = int(tmp_line[3]) ref_base = tmp_line[2].upper() alt_base = dSNPs[chr, pos][1] if ref_base == dSNPs[chr, pos][0]: if cov > 0: reads_bases = tmp_line[4] reads_bases_parsed = parseReadsBases(ref_base, alt_base, reads_bases) fOUT.write("%s\t%d\t%s\t%s\t%s\n" %(chr, pos, ref_base, alt_base, reads_bases_parsed)) # reads_bases_parsed, nReadsBases, nRefBases, dMultiBases, dIndels = parseReadsBases(reads_bases, ref_base, alt_base) # the following is a checkup on other alleles # at this moment this checkup is inactive # number of alleles (SNPs, Indels) other than the alternative allele # nMultiBases = sum(dMultiBases.values()) + sum(dIndels.values()) # if (nReadsBases == nRefBases or # (nMultiBases)/float(nReadsBases) <= 0.05): # out.write("%s\t%d\t%s\t%s\t%s\n" %(chr, pos, ref_base, alt_base, # reads_bases_parsed)) # else: # nRemoved += 1 # print pos, ref_base, alt_base, reads_bases, reads_bases_parsed # print dMultiBases # print dIndels # print else: sys.stderr.write("reference base not consistent\n") sys.stderr.write(line) sys.exit() del dSNPs[chr, pos] fOUT.close()
def getSNPs(isnp): ''' getting polymorphic sties from file file format: 1. chr name 2. pos 3. ref allele 4. alt allele ''' check_if_files_exist(isnp) dSNPs = collections.defaultdict(tuple) with open(isnp, 'r') as fSNPS: for line in fSNPS: tmp_line = line.strip().split("\t") chr = tmp_line[0] pos = int(tmp_line[1]) refBase = tmp_line[2] altBase = tmp_line[3] if not (chr, pos) in dSNPs: dSNPs[chr, pos] = (refBase, altBase) return dSNPs
def getSNPs(isnp): ''' getting polymorphic sties from file file format: 1. chr name 2. pos 3. ref allele 4. alt allele ''' check_if_files_exist(isnp) dSNPs = collections.defaultdict(tuple) with open(isnp, 'r') as fSNPS: for line in fSNPS: tmp_line = line.strip().split("\t") chr = tmp_line[0] pos = int(tmp_line[1]) refBase = tmp_line[2] altBase = tmp_line[3] if not (chr, pos) in dSNPs: dSNPs[chr, pos] = (refBase, altBase) return dSNPs
def run_merge(args): ''' combine allele counts across replicates ''' allele_counts = collections.defaultdict(list) data = collections.defaultdict(list) for ac_file in args.acs: sz_utils.check_if_files_exist(ac_file) ColorText().info( "[poolseq_tk] reading and updating allele counts from %s ..." % (ac_file), "stderr") with open(ac_file) as fAC: for line in fAC: tmp_line = line.strip().split() pos = int(tmp_line[1]) if not pos in data: data[pos] = tmp_line[0:4] if not pos in allele_counts: allele_counts[pos] = map(int, tmp_line[4].split(':')) else: allele_counts[pos] = map( sum, zip(allele_counts[pos], map(int, tmp_line[4].split(':')))) ColorText().info(" [done]\n", "stderr") # output to file fOUT = None if args.out == sys.stdout: fOUT = sys.stdout else: sz_utils.make_dirs_if_necessary(args.out) fOUT = open(args.out, 'w') ColorText().info("[poolseq_tk] outputting to %s ..." % (fOUT.name), "stderr") for pos in sorted(allele_counts.iterkeys()): fOUT.write( "%s\t%s\n" % ("\t".join(data[pos]), ":".join(map(str, allele_counts[pos])))) ColorText().info(" [done]\n", "stderr")
def run_overlap(args): ''' getting SNPs identified from both pools ''' sz_utils.check_if_files_exist(args.file_a, args.file_b) snp_a = collections.defaultdict(list) with open(args.file_a, 'r') as fA: for line in fA: tmp_line = line.strip().split("\t") snp_a[int(tmp_line[1])] = tmp_line ColorText().info("[poolseq_tk]: %d SNPs parsed from %s\n" %(len(snp_a), os.path.basename(args.file_a)), "stderr") sz_utils.make_dirs_if_necessary(args.out) num_overlapion = 0 with open(args.out, 'w') as fOUT: with open(args.file_b, 'r') as fB: for line in fB: tmp_line = line.strip().split("\t") if int(tmp_line[1]) in snp_a: num_overlapion += 1 fOUT.write("%s\t%s\n" %("\t".join(snp_a[int(tmp_line[1])]), "\t".join(tmp_line[-4:]))) ColorText().info("[poolseq_tk]: %d SNPs identified from both pools\n" %(num_overlapion), "stderr")
def run_count(args): ''' Counting alleles at each SNP in the given pileup files ''' dPos = {} if args.pos: ColorText().info("[poolseq_tk] reading SNPs positions:", "stderr") with open(args.pos, 'r') as fPOS: for line in fPOS: tmp_line = line.strip().split("\t") chr = tmp_line[0] pos = int(tmp_line[1]) if (chr, pos) not in dPos: dPos[chr, pos] = 1 ColorText().info(" %d\n" % (len(dPos)), "stderr") else: ColorText().info( "[poolseq_tk] no SNP positions provided ... [skipped]\n", "stderr") ac = collections.defaultdict(tuple) for pileup in args.pileups: sz_utils.check_if_files_exist(pileup) nsnps = 0 ColorText().info( "[poolseq_tk] counting alleles in %s:" % (os.path.basename(pileup)), "stderr") with open(pileup, 'r') as fMPILEUP: for line in fMPILEUP: nsnps += 1 tmp_line = line.strip().split("\t") chr = tmp_line[0] pos = int(tmp_line[1]) if (((chr, pos) in dPos and args.pos) or (len(dPos) == 0 and not args.pos)): ref_base = tmp_line[2] alt_base = tmp_line[3] nRefAlleles, nAltAlleles = 0, 0 if len(tmp_line) == 5: nRefAlleles = tmp_line[-1].count(ref_base) + \ tmp_line[-1].count(ref_base.lower()) nAltAlleles = tmp_line[-1].count(alt_base) + \ tmp_line[-1].count(alt_base.lower()) if (chr, pos) not in ac: ac[chr, pos] = [ ref_base, alt_base, str(nRefAlleles), str(nAltAlleles) ] else: ac[chr, pos] += [str(nRefAlleles), str(nAltAlleles)] ColorText().info(" %d SNPs parsed\n" % (nsnps), "stderr") fOUT = None if args.out == sys.stdout: fOUT = sys.stdout else: sz_utils.make_dirs_if_necessary(args.out) fOUT = open(args.out, 'w') ColorText().info("[poolseq_tk] outputting allele counts to table ...", "stderr") for k in sorted(ac.iterkeys()): chr = k[0] pos = k[1] i = 2 if len(ac[k][i:]) == 2 * len(args.pileups): fOUT.write("%s\t%d\t%s" % (chr, pos, "\t".join(ac[k][0:2]))) while i <= len(ac[k]) - 4: fOUT.write("\t%s" % (":".join(ac[k][i:i + 4]))) i += 4 fOUT.write("\n") ColorText().info(" [done]\n", "stderr") fOUT.close()
def run_fisher(args): ''' run Fisher's Exact test ''' sz_utils.make_dirs_if_necessary(args.outp) sz_utils.check_if_files_exist(args.ac_file) tables = sz_utils._count2table(args.ac_file)[0] task_q = mp.JoinableQueue() result_q = mp.Queue() create_procs(args.nproc, task_q, result_q, args.outp) sz_utils._assign_tables(tables, task_q, args.nproc) try: task_q.join() except KeyboardInterrupt: ColorText().info("[poolseq_tk]: Terminated unexpectedly by keyboard\n", "stderr") sys.exit() else: pvals, odds_ratios, log10_pvals = {}, {}, {} while args.nproc: file = result_q.get() with open(file, 'r') as fIN: for line in fIN: tmp_line = line.strip().split("\t") chr = tmp_line[0] pos = int(tmp_line[1]) pval = float(tmp_line[2]) odds_ratio = float(tmp_line[3]) log10_pval = tmp_line[4] if (chr, pos) not in pvals: pvals[chr, pos] = pval if (chr, pos) not in odds_ratios: odds_ratios[chr, pos] = odds_ratio if (chr, pos) not in log10_pvals: log10_pvals[chr, pos] = log10_pval os.remove(file) # pvals_split, odds_ratios_split = result_q.get() # pvals.update(pvals_split) # odds_ratios.update(odds_ratios_split) args.nproc -= 1 ColorText().info( "[poolseq_tk]: Running Fisher's Exact tests successfully\n", "stderr") # correcting raw p-values and make QQ plots ColorText().info( "[poolseq_tk]: multi-testing correction using %s method at %d%% level ..." % (args.adj_method, args.adj_cutoff * 100), "stderr") raw_pvals = [pvals[k] for k in sorted(pvals.iterkeys())] raw_pvals_vector = robjects.FloatVector(raw_pvals) padjust = robjects.r['p.adjust'](raw_pvals_vector, method=args.adj_method) ColorText().info(" [done]\n", "stderr") ColorText().info( "[poolseq_tk]: p-value cutoff using Benjamini.Hochberg procedure %.5e" % (sz_utils.getFDR_BH(pvals, args.adj_cutoff)), "stderr") ColorText().info(" [done]\n", "stderr") # output p-values ColorText().info("[poolseq_tk]: output to files ...", "stderr") out_all = args.outp + ".fisher.all" out_fdr = args.outp + ".fisher.fdr%d" % (args.adj_cutoff * 100) out_expect = args.outp + ".fisher.fdr%d.expect" % (args.adj_cutoff * 100) with open(out_all, 'w') as fALL, \ open(out_fdr, 'w') as fFDR, \ open(out_expect, 'w') as fEXPECT: for i, k in enumerate(sorted(pvals.iterkeys())): chr = k[0] pos = k[1] raw_pval = pvals[k] log_pval = log10_pvals[k] odds_ratio = odds_ratios[k] if padjust[i] <= args.adj_cutoff: sz_utils._results_outputter(fFDR, pos, chr, "\t".join(tables[k][1:3]), tables[k][3:], raw_pval, log_pval, padjust[i], odds_ratio) if ((args.oddsr_direction == "greater" and odds_ratios[k] > 1) or (args.oddsr_direction == "less" and odds_ratios[k] < 1)): sz_utils._results_outputter(fEXPECT, pos, chr, "\t".join(tables[k][1:3]), tables[k][3:], raw_pval, log_pval, padjust[i], odds_ratio) sz_utils._results_outputter(fALL, pos, chr, "\t".join(tables[k][1:3]), tables[k][3:], raw_pval, log_pval, padjust[i], odds_ratio) ColorText().info(" [done]\n", "stderr") ColorText().info("[poolseq_tk]: Program finishes successfully\n", "stderr")