def make_eval(args): pred_clust_F, ref_clust_F = args pred_clusters = GS.Clusters(False) pred_clusters.read_file(pred_clust_F) ref_clusters = GS.Clusters(False) ref_clusters.read_file(ref_clust_F) # utils.clustering_evaluation(train.complexes, pred_clusters, "Train", True) utils.clustering_evaluation(ref_clusters, pred_clusters, "", True)
def make_eval(args): # pred_clust_F, ref_clust_F, ppiF, cutoff, outF = args pred_clust_F, ref_clust_F = args #num_ppis = CS.lineCount(ppiF) pred_clusters = GS.Clusters(False) pred_clusters.read_file(pred_clust_F) ref_clusters = GS.Clusters(False) ref_clusters.read_file(ref_clust_F) # utils.clustering_evaluation(train.complexes, pred_clusters, "Train", True) scores, head = utils.clustering_evaluation(ref_clusters, pred_clusters, "", True)
def rf_cutoff(args): pred_clust_F, ref_clust_F, ppiF, cutoff, outF = args num_ppis = CS.lineCount(ppiF) pred_clusters = GS.Clusters(False) pred_clusters.read_file(pred_clust_F) ref_clusters = GS.Clusters(False) ref_clusters.read_file(ref_clust_F) # utils.clustering_evaluation(train.complexes, pred_clusters, "Train", True) scores, head = utils.clustering_evaluation(ref_clusters, pred_clusters, "", True) outFH = open(outF, "w") outFH.write("%s\t%i\t%i\t%s\n" % (cutoff, num_ppis, len(pred_clusters.complexes), scores)) outFH.close()
def EPIC_eval_fs(args): in_dir, e_dir, scoreF, refF, outF = args ref_clusters = GS.Clusters(False) ref_clusters.read_file(refF) outFH = open(outF, "w") i = 0 allFiles = paths = [os.path.join(in_dir,fn) for fn in next(os.walk(in_dir))[2]] for file in allFiles: if not file.endswith("clust.txt"): continue pred_clusters = GS.Clusters(False) pred_clusters.read_file(file) _, overlap, _ = pred_clusters.get_matching_complexes(ref_clusters) filesplit = file.split(".")[0:4] fs_comp = filesplit[0].split(os.sep)[-1] scores, head = utils.clustering_evaluation(ref_clusters, pred_clusters, "Eval", False) if i == 0: print "FS_code\tCLF\tSE\tFS\tNum_complexes" + "\t".join(np.array(head.split("\t"))[[0,1,6]]) print >> outFH, "FS_code\tCLF\tSE\tFS\tNum_complexes" + "\t".join(np.array(head.split("\t"))[[0,1,6]]) print fs_comp + "\t" + "\t".join(filesplit[1:4]) + "\t"+ str(len(pred_clusters.complexes))+"\t" + "\t".join(np.array(scores.split("\t"))[[0, 1, 6]]) print >> outFH, fs_comp + "\t" + "\t".join(filesplit[1:4]) + "\t"+ str(len(pred_clusters.complexes))+"\t" + "\t".join(np.array(scores.split("\t"))[[0, 1, 6]]) i += 1 outFH.close()
def main(): feature_combination, input_dir, use_rf, num_cores, mode, anno_source, anno_F, target_taxid, refF, output_dir = sys.argv[1:] #Create feature combination if feature_combination == "00000000": sys.exit() scores = [CS.MutualInformation(2), CS.Bayes(3), CS.Euclidiean(), CS.Wcc(), CS.Jaccard(), CS.Poisson(5), CS.Pearson(), CS.Apex()] this_scores = [] for i, feature_selection in enumerate(feature_combination): if feature_selection == "1": this_scores.append(scores[i]) print "\t".join([fs.name for fs in this_scores]) # Initialize CLF use_rf = use_rf == "True" num_cores = int(num_cores) clf = CS.CLF_Wrapper(num_cores, use_rf) # Load elution data foundprots, elution_datas = utils.load_data(input_dir, this_scores) # Generate reference data set if refF == "": all_gs = utils.create_goldstandard(target_taxid, foundprots) else: all_gs = Goldstandard_from_cluster_File(refF, foundprots) all_gs = utils.create_goldstandard(target_taxid, foundprots) #all_gs = Goldstandard_from_cluster_File(refF, foundprots) # sys.exit() scoreCalc = CS.CalculateCoElutionScores(this_scores, elution_datas, output_dir + ".scores.txt", num_cores=num_cores, cutoff= 0.5) # scoreCalc.calculate_coelutionDatas(all_gs) scoreCalc.readTable(output_dir + ".scores.txt", all_gs) print "training ppis: %i" % len(set(scoreCalc.ppiToIndex.keys())) #n_fold cross validation to test the stability of preicted PPIs utils.stability_evaluation(10, all_gs, scoreCalc, clf, output_dir, mode, anno_source, anno_F) sys.exit() #n_fold cross validation to select the best features. n_fold_cross_validation(10, all_gs, scoreCalc, clf, output_dir, mode, anno_source, anno_F) sys.exit() ###### actually predict the network using all data train, eval = all_gs.split_into_holdout_training(set(scoreCalc.ppiToIndex.keys())) print "All comp:%i" % len(all_gs.complexes.complexes) print "Train comp:%i" % len(train.complexes.complexes) print "Eval comp:%i" % len(eval.complexes.complexes) print "Num valid ppis in training pos: %i" % len(train.positive) print "Num valid ppis in training neg: %i" % len(train.negative) print "Num valid ppis in eval pos: %i" % len(eval.positive) print "Num valid ppis in eval neg: %i" % len(eval.negative) # Evaluate classifier utils.bench_clf(scoreCalc, train, eval, clf, output_dir, verbose=True) functionalData = "" if mode != "exp": functionalData = utils.get_FA_data(anno_source, anno_F) print functionalData.scores.shape # Predict protein interaction network = utils.make_predictions(scoreCalc, mode, clf, all_gs, functionalData) outFH = open("%s.%s.pred.txt" % (output_dir, mode + anno_source), "w") print >> outFH, "\n".join(network) outFH.close() # Predicting clusters utils.predict_clusters("%s.%s.pred.txt" % (output_dir, mode + anno_source), "%s.%s.clust.txt" % (output_dir, mode + anno_source)) # Evaluating predicted clusters pred_clusters = GS.Clusters(False) pred_clusters.read_file("%s.%s.clust.txt" % (output_dir, mode + anno_source)) # utils.clustering_evaluation(train.complexes, pred_clusters, "Train", True) clusterEvaluationScores = utils.clustering_evaluation(eval.complexes, pred_clusters, "", True) outFH = open("%s.%s.evaluation.txt" % (output_dir, mode + anno_source), "w") head = clusterEvaluationScores[1] cluster_scores = clusterEvaluationScores[0] tmp_head = head.split("\t") tmp_scores = cluster_scores.split("\t") for i in range(len(tmp_head)): outFH.write("%s\t%s" % (tmp_head[i], tmp_scores[i])) outFH.write("\n")
def main(): parser = argparse.ArgumentParser() parser.add_argument( "-s", "--feature_selection", type=str, help= "Select which features to use. This is an 8 position long array of 0 and 1, where each position determines which co-elution feature to use. Features sorted by position are: MI, Bayes, Euclidean, WCC, Jaccard, PCCN, PCC, and Apex. Each default=11101001", default="11101001") parser.add_argument( "input_dir", type=str, help="Directory containing the elution files for each experiment") parser.add_argument( "-t", "--taxid", type=str, help="TAXID to automatically download reference from GO,CORUM,INtACT", default="") parser.add_argument( "-c", "--cluster", type=str, help="Path to file containing protein clsuter reference", default="") parser.add_argument("-p", "--ppi", type=str, help="path to ppi File", default="") parser.add_argument("output_dir", type=str, help="Directory containing the output files") parser.add_argument("-o", "--output_prefix", type=str, help="Prefix name for all output Files", default="Out") parser.add_argument( "-M", "--classifier", type=str, help="Select which classifier to use. Values: RF SVM, default RF", default="RF") parser.add_argument("-n", "--num_cores", type=int, help="Number of cores to be used, default 1", default=1) parser.add_argument( "-m", "--mode", type=str, help= "Run EPIC with experimental, functional, or both evidences. Values: EXP, FA, COMB, default: EXP ", default="EXP") parser.add_argument( "-f", "--fun_anno_source", type=str, help= "Where to get functional annotaiton from. Values: STRING or GM or FILE, default= GM", default="GM") parser.add_argument( "-F", "--fun_anno_file", type=str, help= "Path to File containing functional annotation. This flag needs to be set when using FILE as fun_anno_source.", ) parser.add_argument("-r", "--co_elution_cutoff", type=float, help="Co-elution score cutoff. default 0.5", default=0.5) parser.add_argument( "-R", "--classifier_cutoff", type=float, help="Classifier confidence valye cutoff. default = 0.5", default=0.5) parser.add_argument( "-e", "--elution_max_count", type=int, help= "Removies protein that have a maximal peptide count less than the given value. default = 1", default=1) parser.add_argument( "-E", "--frac_count", type=int, help= "Number of fracrions a protein needs to be measured in. default = 2", default=2) parser.add_argument( "-P", "--precalcualted_score_file", type=str, help= "Path to precalulated scorefile to read scores from for faster rerunning of EPIC. default = None", default="NONE") args = parser.parse_args() args.mode = args.mode.upper() args.fun_anno_source = args.fun_anno_source.upper() #Create feature combination if args.feature_selection == "00000000": print "Select at least one feature" sys.exit() this_scores = utils.get_fs_comb(args.feature_selection) print "\t".join([fs.name for fs in this_scores]) # Initialize CLF use_rf = args.classifier == "RF" clf = CS.CLF_Wrapper(args.num_cores, use_rf) # Load elution data foundprots, elution_datas = utils.load_data(args.input_dir, this_scores, fc=args.frac_count, mfc=args.elution_max_count) # Generate reference data set gs = "" if ((args.taxid != "" and args.ppi != "") or (args.cluster != "" and args.ppi != "")): print "Refernce from cluster and PPI are nor compatiple. Please supply ppi or complex reference, not both!" sys.exit() if args.taxid == "" and args.ppi == "" and args.cluster == "": print "Please supply a reference by setting taxid, cluster, or ppi tag" sys.exit() gs_clusters = [] if (args.taxid != "" and args.cluster == "" and args.ppi == ""): print "Loading clusters from GO, CORUM, and Intact" gs_clusters.extend(utils.get_reference_from_net(args.taxid)) if args.cluster != "": print "Loading complexes from file" if args.mode == "FA": gs_clusters.append(GS.FileClusters(args.cluster, "all")) else: gs_clusters.append(GS.FileClusters(args.cluster, foundprots)) if args.ppi != "": print "Reading PPI file from %s" % args.reference gs = Goldstandard_from_PPI_File(args.ppi, foundprots) print gs_clusters if len(gs_clusters) > 0: gs = utils.create_goldstandard(gs_clusters, args.taxid, foundprots) output_dir = args.output_dir + os.sep + args.output_prefix refFH = open(output_dir + ".ref_complexes.txt", "w") for comp in gs.complexes.complexes: print >> refFH, "%s\t%s" % (",".join(comp), ",".join( gs.complexes.complexes[comp])) refFH.close() scoreCalc = CS.CalculateCoElutionScores(this_scores, elution_datas, output_dir + ".scores.txt", num_cores=args.num_cores, cutoff=args.co_elution_cutoff) if args.precalcualted_score_file == "NONE": scoreCalc.calculate_coelutionDatas(gs) else: scoreCalc.readTable(args.precalcualted_score_file, gs) print scoreCalc.scores.shape functionalData = "" gs.positive = set(gs.positive & set(scoreCalc.ppiToIndex.keys())) gs.negative = set(gs.negative & set(scoreCalc.ppiToIndex.keys())) gs.rebalance() print len(gs.positive) print len(gs.negative) if args.mode != "EXP": print "Loading functional data" functionalData = utils.get_FA_data(args.fun_anno_source, args.taxid, args.fun_anno_file) print "Dimension of fun anno " + str(functionalData.scores.shape) print "Start benchmarking" if args.mode == "EXP": utils.cv_bench_clf(scoreCalc, clf, gs, output_dir, format="pdf", verbose=True, folds=5) if args.mode == "COMB": tmp_sc = copy.deepcopy(scoreCalc) tmp_sc.add_fun_anno(functionalData) utils.cv_bench_clf(tmp_sc, clf, gs, output_dir, format="pdf", verbose=True, folds=5) if args.mode == "FA": utils.cv_bench_clf(functionalData, clf, gs, output_dir, format="pdf", verbose=True, folds=5) # PPI evaluation print utils.cv_bench_clf(scoreCalc, clf, gs, args.output_dir, verbose=False, format="pdf", folds=5) #print "I am here" network = utils.make_predictions(scoreCalc, args.mode, clf, gs, fun_anno=functionalData) # Predict protein interaction outFH = open("%s.pred.txt" % (output_dir), "w") final_network = [] for PPI in network: items = PPI.split("\t") if float(items[2]) >= args.classifier_cutoff: final_network.append(PPI) print >> outFH, "\n".join(final_network) outFH.close() # Predicting clusters utils.predict_clusters("%s.pred.txt" % (output_dir), "%s.clust.txt" % (output_dir)) # Evaluating predicted clusters pred_clusters = GS.Clusters(False) pred_clusters.read_file("%s.clust.txt" % (output_dir)) overlapped_complexes_with_reference = gs.get_complexes( ).get_overlapped_complexes_set(pred_clusters) print "# of complexes in reference dataset: " + str( len(overlapped_complexes_with_reference)) #clust_scores, header = utils.clustering_evaluation(gs.complexes, pred_clusters, "", False) clust_scores, header, composite_score = utils.clustering_evaluation( gs.complexes, pred_clusters, "", False) outFH = open("%s.eval.txt" % (output_dir), "w") header = header.split("\t") clust_scores = clust_scores.split("\t") for i, head in enumerate(header): print "%s\t%s" % (head, clust_scores[i]) print >> outFH, "%s\t%s" % (head, clust_scores[i]) outFH.close()
def n_fold_cross_validation(n_fold, all_gs, scoreCalc, clf, output_dir ): tmp_train_eval_container = all_gs.n_fols_split(n_fold) #(all_gs.split_into_n_fold2(n_fold, set(scoreCalc.ppiToIndex.keys()))["turpleKey"]) # tmp_train_eval_container = (all_gs.split_into_n_fold2(n_fold, set(scoreCalc.ppiToIndex.keys()))["turpleKey"]) #the global cluster will contain all clusters predcited from n-fold-corss validation pred_all_clusters = GS.Clusters(False) pred_all_ppis = set([]) complex_count = 0 for index in range(n_fold): print "processinng fold " + str(index + 1) train, eval = tmp_train_eval_container[index] print "All comp:%i" % len(all_gs.complexes.complexes) print "Train comp:%i" % len(train.complexes.complexes) print "Eval comp:%i" % len(eval.complexes.complexes) print "Num valid ppis in training pos: %i" % len(train.positive) print "Num valid ppis in training neg: %i" % len(train.negative) print "Num valid ppis in eval pos: %i" % len(eval.positive) print "Num valid ppis in eval neg: %i" % len(eval.negative) # Evaluate classifier # utils.bench_clf(scoreCalc, train, eval, clf, output_dir, verbose=True) # Predict protein interaction based on n_fold cross validation network = utils.make_predictions_cross_validation(scoreCalc, train, eval, clf) if len(network) == 0: print "No edges were predicted" continue for ppi in network: prota, protb, score = ppi.split("\t") edge = "\t".join(sorted([prota, protb])) pred_all_ppis.add(edge) netF = "%s.fold_%s.pred.txt" % (output_dir, index) clustF = "%s.fold_%s.clust.txt" % (output_dir, index) outFH = open(netF, "w") print >> outFH, "\n".join(network) outFH.close() # Predicting clusters utils.predict_clusters(netF, clustF) # Evaluating predicted clusters pred_clusters = GS.Clusters(False) pred_clusters.read_file(clustF) tmp_complexes_dict = pred_clusters.get_complexes() for key in tmp_complexes_dict: pred_all_clusters.addComplex(complex_count, tmp_complexes_dict[key]) complex_count = complex_count + 1 pred_all_clusters.merge_complexes() print "number of complexes" print len(pred_all_clusters.get_complexes()) print "number of ppis" print len(pred_all_ppis) out_scores, out_head= "%i\t%i\t" % (len(pred_all_ppis), len(pred_all_clusters.get_complexes())), "Num_pred_PPIS\tNUM_pred_CLUST\t" if len(pred_all_clusters.complexes)>0: scores, head = utils.clustering_evaluation(all_gs.complexes, pred_all_clusters, "", True) out_scores += scores out_head += head return out_scores, out_head
def n_fold_cross_validation(n_fold, all_gs, scoreCalc, clf, output_dir, overlap, local): out_scores = [] out_head = [] header = [ "Num_pred_PPIS", "NUM_pred_CLUST", "mmr", "overlapp", "simcoe", "mean_simcoe_overlap", "sensetivity", "ppv", "accuracy", "sep" ] train_eval_container = all_gs.n_fols_split(n_fold, overlap) # create a matrix to store the computed complexes vealuation metrics complex_eval_score_vector = np.zeros((n_fold, 10)) val_ppis = set(scoreCalc.ppiToIndex.keys()) print "Number of ppis with e-score>0.5: %i" % len(val_ppis) #the global cluster will contain all clusters predcited from n-fold-corss validation for index in range(n_fold): print "processinng fold " + str(index + 1) train, eval = train_eval_container[index] train.positive = train.positive & val_ppis train.negative = train.negative & val_ppis train.rebalance() print "All comp:%i" % len(all_gs.complexes.complexes) print "Train comp:%i" % len(train.complexes.complexes) print "Eval comp:%i" % len(eval.complexes.complexes) print "Num valid ppis in training pos: %i" % len(train.positive & val_ppis) print "Num valid ppis in training neg: %i" % len(train.negative & val_ppis) print "Num valid ppis in eval pos: %i" % len(eval.positive) print "Num valid ppis in eval neg: %i" % len(eval.negative) print "Overlap positive %i" % (len(train.positive & eval.positive)) print "Overlap negative %i" % (len(train.negative & eval.negative)) network = [] if local: # Predict protein interaction based on n_fold cross validation network = utils.make_predictions_cross_validation( scoreCalc, train, eval, clf) else: network = utils.predictInteractions(scoreCalc, clf, train, verbose=True) netF = "%s.fold_%s.pred.txt" % (output_dir, index) clustF = "%s.fold_%s.clust.txt" % (output_dir, index) #if os.path.isfile(netF): # netFH = open(netF) # for line in netFH: # line = line.rstrip() # network.append(line) # netFH.close() fold_head = [] if len(network) == 0: print "No edges were predicted" tmp_scores = [0] * 10 fold_head = "\t".join( ["%s%s" % ("Fold %i " % (index + 1), h) for h in header]) out_head.append(fold_head) out_scores.append("\t".join(map(str, tmp_scores))) complex_eval_score_vector[index, :] = tmp_scores continue tmp = [] for ppi in network: prota, protb, score = ppi.split("\t") if float(score) > 0.5: # this is random forest confidence cut off tmp.append(ppi) network = tmp outFH = open(netF, "w") print >> outFH, "\n".join(network) outFH.close() # Predicting clusters utils.predict_clusters(netF, clustF) # Evaluating predicted clusters pred_clusters = GS.Clusters(False) pred_clusters.read_file(clustF) print "number of complexes" print len(pred_clusters.get_complexes()) print "number of ppis" print len(network) fold_scores, fold_head = utils.clustering_evaluation( eval.complexes, pred_clusters, "Fold %i " % (index + 1), True) out_scores.append( "%i\t%i\t%s" % (len(network), len(pred_clusters.get_complexes()), fold_scores)) out_head.append("\t".join( ["%s%s" % ("Fold %i " % (index + 1), h) for h in header])) tmp_scores = [len(network), len(pred_clusters.get_complexes())] tmp_scores.extend(map(float, fold_scores.split("\t"))) tmp_scores = np.array(tmp_scores) complex_eval_score_vector[index, :] = tmp_scores averaged_complex_eval_metrics_vector = np.mean(complex_eval_score_vector, axis=0) out_scores.append("\t".join(map(str, averaged_complex_eval_metrics_vector))) mean_head = "\t".join(["%s%s" % ("Mean ", h) for h in header]) out_head.append(mean_head) return "\t".join(out_scores), "\t".join(out_head)