def preprocess_data(self, position, data): (K,N) = data.shape if self.normalization != "nonorm": self.transit_message("Normalizing using: %s" % self.normalization) (data, factors) = norm_tools.normalize_data(data, self.normalization, self.ctrldata+self.expdata, self.annotation_path) if self.LOESS: self.transit_message("Performing LOESS Correction") for j in range(K): data[j] = stat_tools.loess_correction(position, data[j]) return data
def preprocess_data(self, position, data): (K,N) = data.shape if self.normalization != "nonorm": self.transit_message("Normalizing using: %s" % self.normalization) (data, factors) = norm_tools.normalize_data(data, self.normalization, self.ctrldata+self.expdata, self.annotation_path) if self.LOESS: self.transit_message("Performing LOESS Correction") for j in range(K): data[j] = stat_tools.loess_correction(position, data[j]) return data
def Run(self): #if not self.wxobj: # # Force matplotlib to use good backend for png. # import matplotlib.pyplot as plt #elif "matplotlib.pyplot" not in sys.modules: try: import matplotlib.pyplot as plt except: print "Error: cannot do histograms" self.doHistogram = False self.transit_message("Starting resampling Method") start_time = time.time() if self.doHistogram: histPath = os.path.join( os.path.dirname(self.output.name), transit_tools.fetch_name(self.output.name) + "_histograms") if not os.path.isdir(histPath): os.makedirs(histPath) else: histPath = "" Kctrl = len(self.ctrldata) Kexp = len(self.expdata) #Get orf data self.transit_message("Getting Data") (data, position) = transit_tools.get_validated_data(self.ctrldata + self.expdata, wxobj=self.wxobj) (K, N) = data.shape if self.normalization != "nonorm": self.transit_message("Normalizing using: %s" % self.normalization) (data, factors) = norm_tools.normalize_data(data, self.normalization, self.ctrldata + self.expdata, self.annotation_path) if self.LOESS: self.transit_message("Performing LOESS Correction") for j in range(K): data[j] = stat_tools.loess_correction(position, data[j]) G = tnseq_tools.Genes(self.ctrldata + self.expdata, self.annotation_path, ignoreCodon=self.ignoreCodon, nterm=self.NTerminus, cterm=self.CTerminus, data=data, position=position) #G = tnseq_tools.Genes(self.ctrldata+self.expdata, self.annotation_path, norm=self.normalization, ignoreCodon=self.ignoreCodon, nterm=self.NTerminus, cterm=self.CTerminus) #Resampling data = [] N = len(G) count = 0 self.progress_range(N) for gene in G: count += 1 if gene.k == 0 or gene.n == 0: (test_obs, mean1, mean2, log2FC, pval_ltail, pval_utail, pval_2tail, testlist, data1, data2) = (0, 0, 0, 0, 1.00, 1.00, 1.00, [], [0], [0]) else: if not self.includeZeros: ii = numpy.sum(gene.reads, 0) > 0 else: ii = numpy.ones(gene.n) == 1 data1 = gene.reads[:Kctrl, ii].flatten() + self.pseudocount data2 = gene.reads[Kctrl:, ii].flatten() + self.pseudocount (test_obs, mean1, mean2, log2FC, pval_ltail, pval_utail, pval_2tail, testlist) = stat_tools.resampling( data1, data2, S=self.samples, testFunc=stat_tools.F_mean_diff_flat, adaptive=self.adaptive) if self.doHistogram: import matplotlib.pyplot as plt if testlist: n, bins, patches = plt.hist(testlist, density=1, facecolor='c', alpha=0.75, bins=100) else: n, bins, patches = plt.hist([0, 0], density=1, facecolor='c', alpha=0.75, bins=100) plt.xlabel('Delta Mean') plt.ylabel('Probability') plt.title('%s - Histogram of Delta Mean' % gene.orf) plt.axvline(test_obs, color='r', linestyle='dashed', linewidth=3) plt.grid(True) genePath = os.path.join(histPath, gene.orf + ".png") if not os.path.exists(histPath): os.makedirs(histPath) plt.savefig(genePath) plt.clf() sum1 = numpy.sum(data1) sum2 = numpy.sum(data2) data.append([ gene.orf, gene.name, gene.desc, gene.n, mean1, mean2, sum1, sum2, test_obs, log2FC, pval_2tail ]) # Update progress text = "Running Resampling Method... %5.1f%%" % (100.0 * count / N) self.progress_update(text, count) # self.transit_message("") # Printing empty line to flush stdout self.transit_message("Performing Benjamini-Hochberg Correction") data.sort() qval = stat_tools.BH_fdr_correction([row[-1] for row in data]) self.output.write("#Resampling\n") if self.wxobj: members = sorted([ attr for attr in dir(self) if not callable(getattr(self, attr)) and not attr.startswith("__") ]) memberstr = "" for m in members: memberstr += "%s = %s, " % (m, getattr(self, m)) self.output.write( "#GUI with: norm=%s, samples=%s, pseudocounts=%1.2f, adaptive=%s, histogram=%s, includeZeros=%s, output=%s\n" % (self.normalization, self.samples, self.pseudocount, self.adaptive, self.doHistogram, self.includeZeros, self.output.name.encode('utf-8'))) else: self.output.write("#Console: python %s\n" % " ".join(sys.argv)) self.output.write("#Control Data: %s\n" % (",".join(self.ctrldata).encode('utf-8'))) self.output.write("#Experimental Data: %s\n" % (",".join(self.expdata).encode('utf-8'))) self.output.write("#Annotation path: %s\n" % (self.annotation_path.encode('utf-8'))) self.output.write("#Time: %s\n" % (time.time() - start_time)) self.output.write("#%s\n" % "\t".join(columns)) for i, row in enumerate(data): (orf, name, desc, n, mean1, mean2, sum1, sum2, test_obs, log2FC, pval_2tail) = row self.output.write( "%s\t%s\t%s\t%d\t%1.1f\t%1.1f\t%1.2f\t%1.1f\t%1.2f\t%1.1f\t%1.5f\t%1.5f\n" % (orf, name, desc, n, mean1, mean2, log2FC, sum1, sum2, test_obs, pval_2tail, qval[i])) self.output.close() self.transit_message("Adding File: %s" % (self.output.name)) self.add_file(filetype="Resampling") self.finish() self.transit_message("Finished resampling Method")
def Run(self): self.transit_message("Starting HMM Method") start_time = time.time() #Get data self.transit_message("Getting Data") (data, position) = transit_tools.get_validated_data(self.ctrldata, wxobj=self.wxobj) (K, N) = data.shape # Normalize data if self.normalization != "nonorm": self.transit_message("Normalizing using: %s" % self.normalization) (data, factors) = norm_tools.normalize_data(data, self.normalization, self.ctrldata, self.annotation_path) # Do LOESS if self.LOESS: self.transit_message("Performing LOESS Correction") for j in range(K): data[j] = stat_tools.loess_correction(position, data[j]) hash = transit_tools.get_pos_hash(self.annotation_path) rv2info = transit_tools.get_gene_info(self.annotation_path) if len(self.ctrldata) > 1: self.transit_message("Combining Replicates as '%s'" % self.replicates) O = tnseq_tools.combine_replicates( data, method=self.replicates ) + 1 # Adding 1 to because of shifted geometric in scipy #Parameters Nstates = 4 label = {0: "ES", 1: "GD", 2: "NE", 3: "GA"} reads = O - 1 reads_nz = sorted(reads[reads != 0]) size = len(reads_nz) mean_r = numpy.average(reads_nz[:int(0.95 * size)]) mu = numpy.array([1 / 0.99, 0.01 * mean_r + 2, mean_r, mean_r * 5.0]) #mu = numpy.array([1/0.99, 0.1 * mean_r + 2, mean_r, mean_r*5.0]) L = 1.0 / mu B = [] # Emission Probability Distributions for i in range(Nstates): B.append(scipy.stats.geom(L[i]).pmf) pins = self.calculate_pins(O - 1) pins_obs = sum([1 for rd in O if rd >= 2]) / float(len(O)) pnon = 1.0 - pins pnon_obs = 1.0 - pins_obs for r in range(100): if pnon**r < 0.01: break A = numpy.zeros((Nstates, Nstates)) a = math.log1p(-B[int(Nstates / 2)](1)**r) b = r * math.log(B[int(Nstates / 2)](1)) + math.log( 1.0 / 3) # change to Nstates-1? for i in range(Nstates): A[i] = [b] * Nstates A[i][i] = a PI = numpy.zeros(Nstates) # Initial state distribution PI[0] = 0.7 PI[1:] = 0.3 / (Nstates - 1) self.progress_range(self.maxiterations) ############### ### VITERBI ### (Q_opt, delta, Q) = self.viterbi(A, B, PI, O) ############### ################## ### ALPHA PASS ### (log_Prob_Obs, alpha, C) = self.forward_procedure(numpy.exp(A), B, PI, O) ################## ################# ### BETA PASS ### beta = self.backward_procedure(numpy.exp(A), B, PI, O, C) ################# T = len(O) total = 0 state2count = dict.fromkeys(range(Nstates), 0) for t in range(T): state = Q_opt[t] state2count[state] += 1 total += 1 self.output.write("#HMM - Sites\n") self.output.write("# Tn-HMM\n") if self.wxobj: members = sorted([ attr for attr in dir(self) if not callable(getattr(self, attr)) and not attr.startswith("__") ]) memberstr = "" for m in members: memberstr += "%s = %s, " % (m, getattr(self, m)) self.output.write( "#GUI with: ctrldata=%s, annotation=%s, output=%s\n" % (",".join(self.ctrldata).encode('utf-8'), self.annotation_path.encode('utf-8'), self.output.name.encode('utf-8'))) else: self.output.write("#Console: python3 %s\n" % " ".join(sys.argv)) self.output.write("# \n") self.output.write("# Mean:\t%2.2f\n" % (numpy.average(reads_nz))) self.output.write("# Median:\t%2.2f\n" % numpy.median(reads_nz)) self.output.write("# Normalization:\t%s\n" % self.normalization) self.output.write("# LOESS Correction:\t%s\n" % str(self.LOESS)) self.output.write("# pins (obs):\t%f\n" % pins_obs) self.output.write("# pins (est):\t%f\n" % pins) self.output.write("# Run length (r):\t%d\n" % r) self.output.write("# State means:\n") self.output.write("# %s\n" % " ".join( ["%s: %8.4f" % (label[i], mu[i]) for i in range(Nstates)])) self.output.write("# Self-Transition Prob:\n") self.output.write("# %s\n" % " ".join( ["%s: %2.4e" % (label[i], A[i][i]) for i in range(Nstates)])) self.output.write("# State Emission Parameters (theta):\n") self.output.write("# %s\n" % " ".join( ["%s: %1.4f" % (label[i], L[i]) for i in range(Nstates)])) self.output.write("# State Distributions:") self.output.write("# %s\n" % " ".join([ "%s: %2.2f%%" % (label[i], state2count[i] * 100.0 / total) for i in range(Nstates) ])) states = [int(Q_opt[t]) for t in range(T)] last_orf = "" for t in range(T): s_lab = label.get(states[t], "Unknown State") gamma_t = (alpha[:, t] * beta[:, t]) / numpy.sum( alpha[:, t] * beta[:, t]) genes_at_site = hash.get(position[t], [""]) genestr = "" if not (len(genes_at_site) == 1 and not genes_at_site[0]): genestr = ",".join([ "%s_(%s)" % (g, rv2info.get(g, "-")[0]) for g in genes_at_site ]) self.output.write("%s\t%s\t%s\t%s\t%s\n" % (int(position[t]), int(O[t]) - 1, "\t".join( ["%-9.2e" % g for g in gamma_t]), s_lab, genestr)) self.output.close() self.transit_message("") # Printing empty line to flush stdout self.transit_message("Finished HMM - Sites Method") self.transit_message("Adding File: %s" % (self.output.name)) self.add_file(filetype="HMM - Sites") #Gene Files self.transit_message("Creating HMM Genes Level Output") genes_path = ".".join(self.output.name.split( ".")[:-1]) + "_genes." + self.output.name.split(".")[-1] tempObs = numpy.zeros((1, len(O))) tempObs[0, :] = O - 1 self.post_process_genes(tempObs, position, states, genes_path) self.transit_message("Adding File: %s" % (genes_path)) self.add_file(path=genes_path, filetype="HMM - Genes") self.finish() self.transit_message("Finished HMM Method")
def Run(self): self.transit_message("Starting Mann-Whitney U-test Method") start_time = time.time() Kctrl = len(self.ctrldata) Kexp = len(self.expdata) #Get orf data self.transit_message("Getting Data") (data, position) = transit_tools.get_validated_data(self.ctrldata+self.expdata, wxobj=self.wxobj) (K,N) = data.shape if self.normalization != "nonorm": self.transit_message("Normalizing using: %s" % self.normalization) (data, factors) = norm_tools.normalize_data(data, self.normalization, self.ctrldata+self.expdata, self.annotation_path) if self.LOESS: self.transit_message("Performing LOESS Correction") for j in range(K): data[j] = stat_tools.loess_correction(position, data[j]) G = tnseq_tools.Genes(self.ctrldata + self.expdata, self.annotation_path, ignoreCodon=self.ignoreCodon, nterm=self.NTerminus, cterm=self.CTerminus, data=data, position=position) #u-test data = [] N = len(G) count = 0 self.progress_range(N) for gene in G: count+=1 if gene.k == 0 or gene.n == 0: (test_obs, mean1, mean2, log2FC, u_stat, pval_2tail) = (0, 0, 0, 0, 0.0, 1.00) else: if not self.includeZeros: ii = numpy.sum(gene.reads,0) > 0 else: ii = numpy.ones(gene.n) == 1 data1 = gene.reads[:Kctrl,ii].flatten() data2 = gene.reads[Kctrl:,ii].flatten() try: u_stat, pval_2tail = scipy.stats.mannwhitneyu(data1, data2, alternative="two-sided") except ValueError as e: u_stat, pval_2tail = 0.0, 1.00 n1 = len(data1) n2 = len(data2) mean1 = 0 if n1 > 0: mean1 = numpy.mean(data1) mean2 = 0 if n2 > 0: mean2 = numpy.mean(data2) try: # Only adjust log2FC if one of the means is zero if mean1 > 0 and mean2 > 0: log2FC = math.log((mean2)/(mean1),2) else: log2FC = math.log((mean2+1.0)/(mean1+1.0),2) except: log2FC = 0.0 #["Orf","Name","Desc","Sites","Mean Ctrl","Mean Exp","log2FC", "U-Statistic","p-value","Adj. p-value"] data.append([gene.orf, gene.name, gene.desc, gene.n, mean1, mean2, log2FC, u_stat, pval_2tail]) # Update Progress text = "Running Mann-Whitney U-test Method... %1.1f%%" % (100.0*count/N) self.progress_update(text, count) # self.transit_message("") # Printing empty line to flush stdout self.transit_message("Performing Benjamini-Hochberg Correction") data.sort() qval = stat_tools.BH_fdr_correction([row[-1] for row in data]) self.output.write("#utest\n") if self.wxobj: members = sorted([attr for attr in dir(self) if not callable(getattr(self,attr)) and not attr.startswith("__")]) memberstr = "" for m in members: memberstr += "%s = %s, " % (m, getattr(self, m)) self.output.write("#GUI with: norm=%s, includeZeros=%s, output=%s\n" % (self.normalization, self.includeZeros, self.output.name.encode('utf-8'))) else: self.output.write("#Console: python %s\n" % " ".join(sys.argv)) self.output.write("#Control Data: %s\n" % (",".join(self.ctrldata).encode('utf-8'))) self.output.write("#Experimental Data: %s\n" % (",".join(self.expdata).encode('utf-8'))) self.output.write("#Annotation path: %s\n" % (self.annotation_path.encode('utf-8'))) self.output.write("#Time: %s\n" % (time.time() - start_time)) self.output.write("#%s\n" % "\t".join(columns)) for i,row in enumerate(data): (orf, name, desc, n, mean1, mean2, log2FC, u_stat, pval_2tail) = row self.output.write("%s\t%s\t%s\t%d\t%1.1f\t%1.1f\t%1.2f\t%1.2f\t%1.5f\t%1.5f\n" % (orf, name, desc, n, mean1, mean2, log2FC, u_stat, pval_2tail, qval[i])) self.output.close() self.transit_message("Adding File: %s" % (self.output.name)) self.add_file(filetype="utest") self.finish() self.transit_message("Finished Mann-Whitney U-test Method")
def Run(self): self.transit_message("Starting Genetic Interactions Method") start_time = time.time() self.output.write("#GI\n") wiglist = self.ctrldataA + self.expdataA + self.ctrldataB + self.expdataB Nwig = len(wiglist) Na1 = len(self.ctrldataA) Nb1 = len(self.expdataA) Na2 = len(self.ctrldataB) Nb2 = len(self.expdataB) # Get data self.transit_message("Getting Data") (data, position) = transit_tools.get_validated_data(wiglist, wxobj=self.wxobj) # Normalize data if specified if self.normalization != "nonorm": self.transit_message("Normalizing using: %s" % self.normalization) (data, factors) = norm_tools.normalize_data(data, self.normalization, wiglist, self.annotation_path) # Do LOESS correction if specified if self.LOESS: self.transit_message("Performing LOESS Correction") for j in range(K): data[j] = stat_tools.loess_correction(position, data[j]) # Get Gene objects for each condition G_A1 = tnseq_tools.Genes([], self.annotation_path, data=data[:Na1], position=position,nterm=self.NTerminus,cterm=self.CTerminus) G_B1 = tnseq_tools.Genes([], self.annotation_path, data=data[Na1:(Na1+Nb1)], position=position,nterm=self.NTerminus,cterm=self.CTerminus) G_A2 = tnseq_tools.Genes([], self.annotation_path, data=data[(Na1+Nb1):(Na1+Nb1+Na2)], position=position,nterm=self.NTerminus,cterm=self.CTerminus) G_B2 = tnseq_tools.Genes([], self.annotation_path, data=data[(Na1+Nb1+Na2):], position=position,nterm=self.NTerminus,cterm=self.CTerminus) means_list_a1 = [] means_list_b1 = [] means_list_a2 = [] means_list_b2 = [] var_list_a1 = [] var_list_a2 = [] var_list_b1 = [] var_list_b2 = [] # Base priors on empirical observations accross genes. for gene in sorted(G_A1): if gene.n > 1: A1_data = G_A1[gene.orf].reads.flatten() B1_data = G_B1[gene.orf].reads.flatten() A2_data = G_A2[gene.orf].reads.flatten() B2_data = G_B2[gene.orf].reads.flatten() means_list_a1.append(numpy.mean(A1_data)) var_list_a1.append(numpy.var(A1_data)) means_list_b1.append(numpy.mean(B1_data)) var_list_b1.append(numpy.var(B1_data)) means_list_a2.append(numpy.mean(A2_data)) var_list_a2.append(numpy.var(A2_data)) means_list_b2.append(numpy.mean(B2_data)) var_list_b2.append(numpy.var(B2_data)) # Priors mu0_A1 = scipy.stats.trim_mean(means_list_a1, 0.01) mu0_B1 = scipy.stats.trim_mean(means_list_b1, 0.01) mu0_A2 = scipy.stats.trim_mean(means_list_a2, 0.01) mu0_B2 = scipy.stats.trim_mean(means_list_b2, 0.01) s20_A1 = scipy.stats.trim_mean(var_list_a1, 0.01) s20_B1 = scipy.stats.trim_mean(var_list_b1, 0.01) s20_A2 = scipy.stats.trim_mean(var_list_a2, 0.01) s20_B2 = scipy.stats.trim_mean(var_list_b2, 0.01) k0=1.0 nu0=1.0 data = [] postprob = [] count = 0 N = len(G_A1) self.progress_range(N) # Perform actual analysis for gene in G_A1: # If there is some data if gene.n > 0: A1_data = G_A1[gene.orf].reads.flatten() B1_data = G_B1[gene.orf].reads.flatten() A2_data = G_A2[gene.orf].reads.flatten() B2_data = G_B2[gene.orf].reads.flatten() # Time-1 Time-2 # # Strain-A A C # # Strain-B B D try: muA1_post, varA1_post = stat_tools.sample_trunc_norm_post(A1_data, self.samples, mu0_A1, s20_A1, k0, nu0) muB1_post, varB1_post = stat_tools.sample_trunc_norm_post(B1_data, self.samples, mu0_B1, s20_B1, k0, nu0) muA2_post, varA2_post = stat_tools.sample_trunc_norm_post(A2_data, self.samples, mu0_A2, s20_A2, k0, nu0) muB2_post, varB2_post = stat_tools.sample_trunc_norm_post(B2_data, self.samples, mu0_B2, s20_B2, k0, nu0) except Exception as e: muA1_post = varA1_post = numpy.ones(self.samples) muB1_post = varB1_post = numpy.ones(self.samples) muA2_post = varA2_post = numpy.ones(self.samples) muB2_post = varB2_post = numpy.ones(self.samples) logFC_A_post = numpy.log2(muA2_post/muA1_post) logFC_B_post = numpy.log2(muB2_post/muB1_post) delta_logFC_post = logFC_B_post - logFC_A_post alpha = 0.05 # Get Bounds of the HDI l_logFC_A, u_logFC_A = stat_tools.HDI_from_MCMC(logFC_A_post, 1-alpha) l_logFC_B, u_logFC_B = stat_tools.HDI_from_MCMC(logFC_B_post, 1-alpha) l_delta_logFC, u_delta_logFC = stat_tools.HDI_from_MCMC(delta_logFC_post, 1-alpha) mean_logFC_A = numpy.mean(logFC_A_post) mean_logFC_B = numpy.mean(logFC_B_post) mean_delta_logFC = numpy.mean(delta_logFC_post) # Is HDI significantly different than ROPE? not_HDI_overlap_bit = l_delta_logFC > self.rope or u_delta_logFC < -self.rope # Probability of posterior overlaping with ROPE probROPE = numpy.mean(numpy.logical_and(delta_logFC_post>=0.0-self.rope, delta_logFC_post<=0.0+self.rope)) # If there is no data, assume empty defaults else: A1_data = [0,0] B1_data = [0,0] A2_data = [0,0] B2_data = [0,0] muA1_post = varA1_post = numpy.ones(self.samples) muB1_post = varB1_post = numpy.ones(self.samples) muA2_post = varA2_post = numpy.ones(self.samples) muB2_post = varB2_post = numpy.ones(self.samples) logFC_A_post = numpy.log2(muA2_post/muA1_post) logFC_B_post = numpy.log2(muB2_post/muB1_post) delta_logFC_post = logFC_B_post - logFC_A_post mean_logFC_A = 0 mean_logFC_B = 0 mean_delta_logFC = 0 l_logFC_A = 0 u_logFC_A = 0 l_logFC_B = 0 u_logFC_B = 0 l_delta_logFC = 0 u_delta_logFC = 0 probROPE = 1.0 if numpy.isnan(l_logFC_A): l_logFC_A = -10 u_logFC_A = 10 if numpy.isnan(l_logFC_B): l_logFC_B = -10 u_logFC_B = 10 if numpy.isnan(l_delta_logFC): l_delta_logFC = -10 u_delta_logFC = 10 postprob.append(probROPE) data.append((gene.orf, gene.name, gene.n, numpy.mean(muA1_post), numpy.mean(muA2_post), numpy.mean(muB1_post), numpy.mean(muB2_post), mean_logFC_A, mean_logFC_B, mean_delta_logFC, l_delta_logFC, u_delta_logFC, probROPE, not_HDI_overlap_bit)) text = "Running GI Method... %2.0f%%" % (100.0*(count+1)/N) self.progress_update(text, count) self.transit_message_inplace("Running Export Method... %1.1f%%" % (100.0*count/(N-1))) count+=1 data.sort(key=lambda x: x[-2]) if self.doBFDR or not self.doFWER: postprob = numpy.array(postprob) postprob.sort() bfdr = numpy.cumsum(postprob)/numpy.arange(1, len(postprob)+1) adjusted_prob = bfdr adjusted_label = "BFDR" elif doFWER: fwer = FWER_Bayes(postprob) fwer.sort() adjusted_prob = fwer adjusted_label = "FWER" # If not using adjustment for classification, sort correctly if not self.doBFDR and not self.doFWER: sorted_index = numpy.argsort([d[-1] for d in data])[::-1][:len(data)] adjusted_prob = [adjusted_prob[ii] for ii in sorted_index] data = [data[ii] for ii in sorted_index] # Print output if self.wxobj: members = sorted([attr for attr in dir(self) if not callable(getattr(self,attr)) and not attr.startswith("__")]) memberstr = "" for m in members: memberstr += "%s = %s, " % (m, getattr(self, m)) self.output.write("#GUI with: norm=%s, samples=%s, includeZeros=%s, output=%s\n" % (self.normalization, self.samples, self.includeZeros, self.output.name.encode('utf-8'))) else: self.output.write("#Console: python %s\n" % " ".join(sys.argv)) self.output.write("#Control Data-A: %s\n" % (",".join(self.ctrldataA).encode('utf-8'))) self.output.write("#Control Data-B: %s\n" % (",".join(self.ctrldataB).encode('utf-8'))) self.output.write("#Experimental Data-A: %s\n" % (",".join(self.expdataA).encode('utf-8'))) self.output.write("#Experimental Data-B: %s\n" % (",".join(self.expdataB).encode('utf-8'))) self.output.write("#Annotation path: %s\n" % (self.annotation_path.encode('utf-8'))) self.output.write("#Time: %s\n" % (time.time() - start_time)) self.output.write("#%s\n" % "\t".join(columns)) if self.doBFDR or self.doFWER: self.output.write("# Significant interactions are those whose adjusted probability of the delta-logFC falling within ROPE is < 0.05 (Adjusted using %s)\n" % (adjusted_label)) else: self.output.write("# Significant interactions are those genes whose delta-logFC HDI does not overlap the ROPE\n") self.output.write("#\n") # Write column names self.output.write("#ORF\tName\tNumber of TA Sites\tMean count (Strain A Time 1)\tMean count (Strain A Time 2)\tMean count (Strain B Time 1)\tMean count (Strain B Time 2)\tMean logFC (Strain A)\tMean logFC (Strain B) \tMean delta logFC\tLower Bound delta logFC\tUpper Bound delta logFC\tProb. of delta-logFC being within ROPE\tAdjusted Probability (%s)\tIs HDI outside ROPE?\tType of Interaction\n" % adjusted_label) # Write gene results for i,row in enumerate(data): #1 2 3 4 5 6 7 8 9 10 11 12 13 14 orf, name, n, mean_muA1_post, mean_muA2_post, mean_muB1_post, mean_muB2_post, mean_logFC_A, mean_logFC_B, mean_delta_logFC, l_delta_logFC, u_delta_logFC, probROPE, not_HDI_overlap_bit = row type_of_interaction = "No Interaction" if ((self.doBFDR or self.doFWER) and adjusted_prob[i] < 0.05): type_of_interaction = self.classify_interaction(mean_delta_logFC, mean_logFC_B, mean_logFC_A) elif not (self.doBFDR or self.doFWER) and not_HDI_overlap_bit: type_of_interaction = self.classify_interaction(mean_delta_logFC, mean_logFC_B, mean_logFC_A) new_row = tuple(list(row[:-1])+[adjusted_prob[i], not_HDI_overlap_bit, type_of_interaction]) self.output.write("%s\t%s\t%d\t%1.2f\t%1.2f\t%1.2f\t%1.2f\t%1.2f\t%1.2f\t%1.2f\t%1.2f\t%1.2f\t%1.8f\t%1.8f\t%s\t%s\n" % new_row) self.transit_message("Adding File: %s" % (self.output.name)) self.add_file(filetype="GI") self.finish() self.transit_message("Finished Genetic Interactions Method")
def Run(self): self.transit_message("Starting Genetic Interactions Method") start_time = time.time() self.output.write("#GI\n") wiglist = self.ctrldataA + self.ctrldataB + self.expdataA + self.expdataB Nwig = len(wiglist) Na1 = len(self.ctrldataA) Nb1 = len(self.ctrldataB) Na2 = len(self.expdataA) Nb2 = len(self.expdataB) # Get data self.transit_message("Getting Data") (data, position) = transit_tools.get_validated_data(wiglist, wxobj=self.wxobj) # Normalize data if specified if self.normalization != "nonorm": self.transit_message("Normalizing using: %s" % self.normalization) (data, factors) = norm_tools.normalize_data(data, self.normalization, wiglist, self.annotation_path) # Do LOESS correction if specified if self.LOESS: self.transit_message("Performing LOESS Correction") for j in range(K): data[j] = stat_tools.loess_correction(position, data[j]) # Get Gene objects for each condition G_A1 = tnseq_tools.Genes([], self.annotation_path, data=data[:Na1], position=position, nterm=self.NTerminus, cterm=self.CTerminus) G_B1 = tnseq_tools.Genes([], self.annotation_path, data=data[Na1:(Na1 + Nb1)], position=position, nterm=self.NTerminus, cterm=self.CTerminus) G_A2 = tnseq_tools.Genes([], self.annotation_path, data=data[(Na1 + Nb1):(Na1 + Nb1 + Na2)], position=position, nterm=self.NTerminus, cterm=self.CTerminus) G_B2 = tnseq_tools.Genes([], self.annotation_path, data=data[(Na1 + Nb1 + Na2):], position=position, nterm=self.NTerminus, cterm=self.CTerminus) means_list_a1 = [] means_list_b1 = [] means_list_a2 = [] means_list_b2 = [] var_list_a1 = [] var_list_a2 = [] var_list_b1 = [] var_list_b2 = [] # Base priors on empirical observations across genes. for gene in sorted(G_A1): if gene.n > 1: A1_data = G_A1[gene.orf].reads.flatten() B1_data = G_B1[gene.orf].reads.flatten() A2_data = G_A2[gene.orf].reads.flatten() B2_data = G_B2[gene.orf].reads.flatten() means_list_a1.append(numpy.mean(A1_data)) var_list_a1.append(numpy.var(A1_data)) means_list_b1.append(numpy.mean(B1_data)) var_list_b1.append(numpy.var(B1_data)) means_list_a2.append(numpy.mean(A2_data)) var_list_a2.append(numpy.var(A2_data)) means_list_b2.append(numpy.mean(B2_data)) var_list_b2.append(numpy.var(B2_data)) # Priors mu0_A1 = scipy.stats.trim_mean(means_list_a1, 0.01) mu0_B1 = scipy.stats.trim_mean(means_list_b1, 0.01) mu0_A2 = scipy.stats.trim_mean(means_list_a2, 0.01) mu0_B2 = scipy.stats.trim_mean(means_list_b2, 0.01) s20_A1 = scipy.stats.trim_mean(var_list_a1, 0.01) s20_B1 = scipy.stats.trim_mean(var_list_b1, 0.01) s20_A2 = scipy.stats.trim_mean(var_list_a2, 0.01) s20_B2 = scipy.stats.trim_mean(var_list_b2, 0.01) k0 = 1.0 nu0 = 1.0 data = [] postprob = [] count = 0 N = len(G_A1) self.progress_range(N) # Perform actual analysis for gene in G_A1: # If there is some data if gene.n > 0: A1_data = G_A1[gene.orf].reads.flatten() B1_data = G_B1[gene.orf].reads.flatten() A2_data = G_A2[gene.orf].reads.flatten() B2_data = G_B2[gene.orf].reads.flatten() # Time-1 Time-2 # # Strain-A A C # # Strain-B B D try: muA1_post, varA1_post = stat_tools.sample_trunc_norm_post( A1_data, self.samples, mu0_A1, s20_A1, k0, nu0) muB1_post, varB1_post = stat_tools.sample_trunc_norm_post( B1_data, self.samples, mu0_B1, s20_B1, k0, nu0) muA2_post, varA2_post = stat_tools.sample_trunc_norm_post( A2_data, self.samples, mu0_A2, s20_A2, k0, nu0) muB2_post, varB2_post = stat_tools.sample_trunc_norm_post( B2_data, self.samples, mu0_B2, s20_B2, k0, nu0) except Exception as e: muA1_post = varA1_post = numpy.ones(self.samples) muB1_post = varB1_post = numpy.ones(self.samples) muA2_post = varA2_post = numpy.ones(self.samples) muB2_post = varB2_post = numpy.ones(self.samples) logFC_A_post = numpy.log2(muA2_post / muA1_post) logFC_B_post = numpy.log2(muB2_post / muB1_post) delta_logFC_post = logFC_B_post - logFC_A_post alpha = 0.05 # Get Bounds of the HDI l_logFC_A, u_logFC_A = stat_tools.HDI_from_MCMC( logFC_A_post, 1 - alpha) l_logFC_B, u_logFC_B = stat_tools.HDI_from_MCMC( logFC_B_post, 1 - alpha) l_delta_logFC, u_delta_logFC = stat_tools.HDI_from_MCMC( delta_logFC_post, 1 - alpha) mean_logFC_A = numpy.mean(logFC_A_post) mean_logFC_B = numpy.mean(logFC_B_post) mean_delta_logFC = numpy.mean(delta_logFC_post) # Is HDI significantly different than ROPE? (i.e. no overlap) not_HDI_overlap_bit = l_delta_logFC > self.rope or u_delta_logFC < -self.rope # Probability of posterior overlaping with ROPE probROPE = numpy.mean( numpy.logical_and(delta_logFC_post >= 0.0 - self.rope, delta_logFC_post <= 0.0 + self.rope)) # If there is no data, assume empty defaults else: A1_data = [0, 0] B1_data = [0, 0] A2_data = [0, 0] B2_data = [0, 0] muA1_post = varA1_post = numpy.ones(self.samples) muB1_post = varB1_post = numpy.ones(self.samples) muA2_post = varA2_post = numpy.ones(self.samples) muB2_post = varB2_post = numpy.ones(self.samples) logFC_A_post = numpy.log2(muA2_post / muA1_post) logFC_B_post = numpy.log2(muB2_post / muB1_post) delta_logFC_post = logFC_B_post - logFC_A_post mean_logFC_A = 0 mean_logFC_B = 0 mean_delta_logFC = 0 l_logFC_A = 0 u_logFC_A = 0 l_logFC_B = 0 u_logFC_B = 0 l_delta_logFC = 0 u_delta_logFC = 0 probROPE = 1.0 if numpy.isnan(l_logFC_A): l_logFC_A = -10 u_logFC_A = 10 if numpy.isnan(l_logFC_B): l_logFC_B = -10 u_logFC_B = 10 if numpy.isnan(l_delta_logFC): l_delta_logFC = -10 u_delta_logFC = 10 postprob.append(probROPE) data.append((gene.orf, gene.name, gene.n, numpy.mean(muA1_post), numpy.mean(muA2_post), numpy.mean(muB1_post), numpy.mean(muB2_post), mean_logFC_A, mean_logFC_B, mean_delta_logFC, l_delta_logFC, u_delta_logFC, probROPE, not_HDI_overlap_bit)) text = "Running GI Method... %2.0f%%" % (100.0 * (count + 1) / N) self.progress_update(text, count) self.transit_message_inplace("Running Export Method... %1.1f%%" % (100.0 * count / (N - 1))) count += 1 # for HDI, maybe I should sort on abs(mean_delta_logFC); however, need to sort by prob to calculate BFDR probcol = -2 # probROPEs data.sort(key=lambda x: x[probcol]) sortedprobs = numpy.array([x[probcol] for x in data]) # BFDR method: Newton M.A., Noueiry A., Sarkar D., Ahlquist P. (2004). Detecting differential gene expression with a semiparametric hierarchical mixture method. Biostatistics, 5:155–176. if self.signif == "BFDR": sortedprobs = numpy.array(sortedprobs) #sortedprobs.sort() # why, since already sorted? bfdr = numpy.cumsum(sortedprobs) / numpy.arange( 1, len(sortedprobs) + 1) adjusted_prob = bfdr # should be same order as sorted above by probROPE adjusted_label = "BFDR" elif self.signif == "FWER": fwer = stat_tools.FWER_Bayes(sortedprobs) #fwer.sort() # should not need this if monotonic adjusted_prob = fwer adjusted_label = "FWER" # If not using adjustment for classification, sort correctly else: adjusted_prob = sortedprobs adjusted_label = "un" # should I stable-sort by overlap_bit? # sorted_index = numpy.argsort([d[-1] for d in data])[::-1][:len(data)] # adjusted_prob = [adjusted_prob[ii] for ii in sorted_index] # data = [data[ii] for ii in sorted_index] # Print(output) if self.wxobj: members = sorted([ attr for attr in dir(self) if not callable(getattr(self, attr)) and not attr.startswith("__") ]) memberstr = "" for m in members: memberstr += "%s = %s, " % (m, getattr(self, m)) self.output.write( "#GUI with: norm=%s, samples=%s, includeZeros=%s, output=%s\n" % (self.normalization, self.samples, self.includeZeros, self.output.name.encode('utf-8'))) else: self.output.write("#Console: python3 %s\n" % " ".join(sys.argv)) now = str(datetime.datetime.now()) now = now[:now.rfind('.')] self.output.write("#Date: " + now + "\n") #self.output.write("#Runtime: %s s\n" % (time.time() - start_time)) self.output.write("#Control Data-A: %s\n" % (",".join(self.ctrldataA).encode('utf-8'))) self.output.write("#Control Data-B: %s\n" % (",".join(self.ctrldataB).encode('utf-8'))) self.output.write("#Experimental Data-A: %s\n" % (",".join(self.expdataA).encode('utf-8'))) self.output.write("#Experimental Data-B: %s\n" % (",".join(self.expdataB).encode('utf-8'))) self.output.write("#Annotation path: %s\n" % (self.annotation_path.encode('utf-8'))) self.output.write("#ROPE=%s, method for significance=%s\n" % (self.rope, self.signif)) #self.output.write("#%s\n" % "\t".join(columns)) if self.signif == "HDI": self.output.write( "#Significant interactions are those genes whose delta-logFC HDI does not overlap the ROPE\n" ) elif self.signif in "prob BDFR FWER": self.output.write( "#Significant interactions are those whose %s-adjusted probability of the delta-logFC falling within ROPE is < 0.05.\n" % (adjusted_label)) # Write column names (redundant with self.columns) self.output.write( "#ORF\tName\tNumber of TA Sites\tMean count (Strain A Condition 1)\tMean count (Strain A Condition 2)\tMean count (Strain B Condition 1)\tMean count (Strain B Condition 2)\tMean logFC (Strain A)\tMean logFC (Strain B) \tMean delta logFC\tLower Bound delta logFC\tUpper Bound delta logFC\tIs HDI outside ROPE?\tProb. of delta-logFC being within ROPE\t%s-Adjusted Probability\tType of Interaction\n" % adjusted_label) # Write gene results for i, row in enumerate(data): #1 2 3 4 5 6 7 8 9 10 11 12 13 14 orf, name, n, mean_muA1_post, mean_muA2_post, mean_muB1_post, mean_muB2_post, mean_logFC_A, mean_logFC_B, mean_delta_logFC, l_delta_logFC, u_delta_logFC, probROPE, not_HDI_overlap_bit = row interaction = self.classify_interaction(mean_delta_logFC, mean_logFC_B, mean_logFC_A) type_of_interaction = "No Interaction" if self.signif in "prob BFDR FWER" and adjusted_prob[i] < 0.05: type_of_interaction = interaction if self.signif == "HDI" and not_HDI_overlap_bit: type_of_interaction = interaction new_row = tuple( list(row[:-2]) + [ not_HDI_overlap_bit, probROPE, adjusted_prob[i], type_of_interaction ]) self.output.write( "%s\t%s\t%d\t%1.2f\t%1.2f\t%1.2f\t%1.2f\t%1.2f\t%1.2f\t%1.2f\t%1.2f\t%1.2f\t%s\t%1.8f\t%1.8f\t%s\n" % new_row) self.transit_message("Adding File: %s" % (self.output.name)) self.add_file(filetype="GI") self.finish() self.transit_message("Finished Genetic Interactions Method")