def _update_metrics(self, y_true, y_pred, onco_prob, tsg_prob): # record which genes were predicted what self.driver_gene_pred = pd.Series(y_pred, self.y.index) self.driver_gene_score = pd.Series(onco_prob+tsg_prob, self.y.index) # evaluate performance prec, recall, fscore, support = metrics.precision_recall_fscore_support(y_true, y_pred, average='macro') cancer_gene_pred = ((onco_prob + tsg_prob)>.5).astype(int) self.cancer_gene_count[self.num_pred] = np.sum(cancer_gene_pred) self.precision[self.num_pred] = prec self.recall[self.num_pred] = recall self.f1_score[self.num_pred] = fscore # compute Precision-Recall curve metrics driver_prob = onco_prob + tsg_prob driver_true = (y_true > 0).astype(int) p, r, thresh = metrics.precision_recall_curve(driver_true, driver_prob) p, r, thresh = p[::-1], r[::-1], thresh[::-1] # reverse order of results thresh = np.insert(thresh, 0, 1.0) self.driver_precision_array[self.num_pred, :] = interp(self.driver_recall_array, r, p) self.driver_threshold_array[self.num_pred, :] = interp(self.driver_recall_array, r, thresh) # calculate prediction summary statistics prec, recall, fscore, support = metrics.precision_recall_fscore_support(driver_true, cancer_gene_pred) self.driver_precision[self.num_pred] = prec[1] self.driver_recall[self.num_pred] = recall[1] # save driver metrics fpr, tpr, thresholds = metrics.roc_curve(driver_true, driver_prob) self.driver_tpr_array[self.num_pred, :] = interp(self.driver_fpr_array, fpr, tpr)
def test(self, a_trees, a_segments): """Estimate performance of segmenter model. Args: a_trees (list): BitPar trees a_segments (list): corresponding gold segments for trees Returns: 2-tuple: macro and micro-averaged F-scores """ if self.model is None: return (0, 0) segments = [self.model.predict(self.featgen(itree))[0] for itree in a_trees] a_segments = [str(s) for s in a_segments] _, _, macro_f1, _ = precision_recall_fscore_support(a_segments, segments, average='macro', warn_for=()) _, _, micro_f1, _ = precision_recall_fscore_support(a_segments, segments, average='micro', warn_for=()) return (macro_f1, micro_f1)
def learnCART(self): train_input_data = self.loadData(self.train_file) target = [x[1] for x in train_input_data] target = target[1:] features = [x[2:] for x in train_input_data] features = features[1:] # feature selection #features_new = self.doFeatureSelection(features,target) model = self.classify(features,target) test_input_data = self.loadData(self.test_file) actualOutput = [x[1] for x in test_input_data] actualOutput = actualOutput[1:] features = [x[2:] for x in test_input_data] features = features[1:] predictedOutput = model.predict(features) #print predictedOutput #print actualOutput self.computeAccuracy(predictedOutput,actualOutput) print "Precision recall Fscore support metrics for CART " print precision_recall_fscore_support(actualOutput,predictedOutput) print "\nconfusion matrix\n" print confusion_matrix(actualOutput,predictedOutput) self.printDTRules(model) X= [] Y=[] for a in predictedOutput: X.append(int(a)) for a in actualOutput: Y.append(int(a)) self.plotROC(Y,X) result = zip(Y,X) self.write_To_File(result,"cart-predictions.csv")
def compare_dummy_classification(self): """ Compares classifier to dummy classifiers. Return results (resultscores_tuple, N.A., N.A.)""" X_train = self.train_vectors y_train = self.train_tweetclasses X_test = self.test_vectors y_test = self.test_tweetclasses dummy_results = [] dummy = DummyClassifier(strategy="most_frequent", random_state=0) dummy.fit(X_train, y_train) y_true, y_preddum = y_test, dummy.predict(X_test) tuples = precision_recall_fscore_support(y_true, y_preddum) dummy1 = DummyClassifier(strategy="stratified", random_state=0) dummy1.fit(X_train, y_train) y_true, y_preddum1 = y_test, dummy1.predict(X_test) tuples1 = precision_recall_fscore_support(y_true, y_preddum1) dummy2 = DummyClassifier(strategy="uniform", random_state=0) dummy2.fit(X_train, y_train) y_true, y_preddum2 = y_test, dummy2.predict(X_test) tuples2 = precision_recall_fscore_support(y_true, y_preddum2) resulttuple = ("dummy freq", "N.A.", "N.A.", "N.A.", "N.A.", tuples) resulttuple1 = ("dummy strat", "N.A.", "N.A.", "N.A.", "N.A.", tuples1) resulttuple2 = ("dummy uni", "N.A.", "N.A.", "N.A.", "N.A.", tuples2) dummy_results.append(resulttuple) dummy_results.append(resulttuple1) dummy_results.append(resulttuple2) return dummy_results
def main(): do_it = 1 # get data global g_train, g_train_label, g_test, g_test_label, g_feature_name g_train, g_train_label, g_test, g_test_label, g_feature_name = load_features_and_labels() # do if do_it == 0: forest = cudaTreeRandomForestClassifier(n_estimators=50, verbose=True, bootstrap=False) forest.fit(np.asarray(g_train), np.asarray(g_train_label), bfs_threshold=4196) predictions = forest.predict(np.asarray(g_test)) print precision_recall_fscore_support(g_test_label, predictions, average='micro') # do if do_it == 0: forest = hybridForestRandomForestClassifier(n_estimators=50, n_gpus=2, n_jobs=6, bootstrap=False, cpu_classifier=WiseRF) forest.fit(np.asarray(g_train), np.asarray(g_train_label), bfs_threshold=4196) predictions = forest.predict(np.asarray(g_test)) print precision_recall_fscore_support(g_test_label, predictions, average='micro')
def compare_dummy(self): """ Compares classifier to dummy classifiers""" #print "\nDetailed classification report:\n" #print "The model is trained on the full development set.\n" #print "The scores are computed on the full evaluation set.\n" X_train = self.train_vectors y_train = self.train_tweetclasses X_test = self.test_vectors y_test = self.test_tweetclasses dummy = DummyClassifier(strategy='most_frequent',random_state=0) dummy.fit(X_train, y_train) y_true, y_preddum = y_test, dummy.predict(X_test) tuples = precision_recall_fscore_support(y_true, y_preddum) dummy1 = DummyClassifier(strategy='stratified',random_state=0) dummy1.fit(X_train, y_train) y_true, y_preddum1 = y_test, dummy1.predict(X_test) tuples1 = precision_recall_fscore_support(y_true, y_preddum1) dummy2 = DummyClassifier(strategy='uniform',random_state=0) dummy2.fit(X_train, y_train) y_true, y_preddum2 = y_test, dummy2.predict(X_test) tuples2 = precision_recall_fscore_support(y_true, y_preddum2) return (tuples, tuples1,tuples2)
def compute_precision_recall_accuracy_thresholded_v3(threshold=0.5, sim_column=4): """ starting from 0 the sims (for supp-v2) are: soundex nysiis metaphone :param threshold: :param sim_column: :return: """ tab_strings, tab_values, ground_truth = read_in_sim_data_as_table() y_true = list() y_pred = list() num_pos = 0 output_pos = 0 for i in range(len(tab_values)): y_true.append(ground_truth[i]) if ground_truth[i] == 1: num_pos += 1 if tab_values[i][sim_column] >= threshold: y_pred.append(1) output_pos += 1 else: y_pred.append(0) # precision, recall, thresholds = precision_recall_curve(np.array(y_true), np.array(y_pred)) print 'printing precision, recall, fscore, support:', print precision_recall_fscore_support(np.array(y_true), np.array(y_pred), average='binary') print 'accuracy score: ', accuracy_score(np.array(y_true), np.array(y_pred)) # print precision # print recall print 'number of positives output: ', print num_pos print 'number of positives in ground truth: ', print output_pos
def compute_precision_recall_accuracy_thresholded_v2(threshold=0.5, sim_column=4): """ starting from 0 the sims (for supp-v2) are: tri_gram_jaccard_similarity jaro_winkler_similarity levenshtein_similarity needleman_wunsch_similarity metaphone_similarity :param threshold: :param sim_column: :return: """ tab_strings, tab_values, ground_truth = read_in_sim_data_as_table() y_true = list() y_pred = list() num_pos = 0 output_pos = 0 for i in range(len(tab_values)): y_true.append(ground_truth[i]) if ground_truth[i] == 1: num_pos += 1 if tab_values[i][sim_column] >= threshold: y_pred.append(1) output_pos += 1 else: y_pred.append(0) # precision, recall, thresholds = precision_recall_curve(np.array(y_true), np.array(y_pred)) print precision_recall_fscore_support(np.array(y_true), np.array(y_pred), average='binary') print 'accuracy score: ',accuracy_score(np.array(y_true), np.array(y_pred)) # print precision # print recall print num_pos print output_pos
def test_precision_recall_f1_score_with_an_empty_prediction(): y_true = np.array([[0, 1, 0, 0], [1, 0, 0, 0], [0, 1, 1, 0]]) y_pred = np.array([[0, 0, 0, 0], [0, 0, 0, 1], [0, 1, 1, 0]]) # true_pos = [ 0. 1. 1. 0.] # false_pos = [ 0. 0. 0. 1.] # false_neg = [ 1. 1. 0. 0.] p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average=None) assert_array_almost_equal(p, [0.0, 1.0, 1.0, 0.0], 2) assert_array_almost_equal(r, [0.0, 0.5, 1.0, 0.0], 2) assert_array_almost_equal(f, [0.0, 1 / 1.5, 1, 0.0], 2) assert_array_almost_equal(s, [1, 2, 1, 0], 2) f2 = fbeta_score(y_true, y_pred, beta=2, average=None) support = s assert_array_almost_equal(f2, [0, 0.55, 1, 0], 2) p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average="macro") assert_almost_equal(p, 0.5) assert_almost_equal(r, 1.5 / 4) assert_almost_equal(f, 2.5 / (4 * 1.5)) assert_equal(s, None) assert_almost_equal(fbeta_score(y_true, y_pred, beta=2, average="macro"), np.mean(f2)) p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average="micro") assert_almost_equal(p, 2 / 3) assert_almost_equal(r, 0.5) assert_almost_equal(f, 2 / 3 / (2 / 3 + 0.5)) assert_equal(s, None) assert_almost_equal(fbeta_score(y_true, y_pred, beta=2, average="micro"), (1 + 4) * p * r / (4 * p + r)) p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average="weighted") assert_almost_equal(p, 3 / 4) assert_almost_equal(r, 0.5) assert_almost_equal(f, (2 / 1.5 + 1) / 4) assert_equal(s, None) assert_almost_equal(fbeta_score(y_true, y_pred, beta=2, average="weighted"), np.average(f2, weights=support)) p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average="samples") # |h(x_i) inter y_i | = [0, 0, 2] # |y_i| = [1, 1, 2] # |h(x_i)| = [0, 1, 2] assert_almost_equal(p, 1 / 3) assert_almost_equal(r, 1 / 3) assert_almost_equal(f, 1 / 3) assert_equal(s, None) assert_almost_equal(fbeta_score(y_true, y_pred, beta=2, average="samples"), 0.333, 2)
def main(): model_file = '../../paper/data/srwe_model/wiki_small.w2v.model' nytimes_file = '../gen_data/nytimes/news_corpus' model = load_w2v_model(model_file, logging, nparray=True) corpus_vec, corpus_label = load_nytimes(nytimes_file, model) labels = list(set(corpus_label)) X_train, X_test, y_train, y_test = train_test_split(corpus_vec, corpus_label, test_size=0.2, random_state=42) logging.info('train size: %d, test size:%d' % (len(y_train), len(y_test))) clfs = {} for label in labels: clfs[label] = train(label, X_train, X_test, y_train, y_test) y_pred = [] for each in X_test: pred_res = [] for label in clfs: pred_res.append((clfs[label].predict_proba(each.reshape(1, -1))[0][1], label)) sorted_pred = sorted(pred_res, key=lambda x: x[0], reverse=True) y_pred.append(sorted_pred[0][1]) precision, recall, f_score, support, present_labels = precision_recall_fscore_support(y_test, y_pred) for l, p, r, f in zip(present_labels, precision, recall, f_score): print '%s\t%.4lf\t%.4lf\t%.4lf' % (l, p, r, f) precision, recall, f_score, support, present_labels = precision_recall_fscore_support(y_test, y_pred, average='macro') print 'Macro\t%.4lf\t%.4lf\t%.4lf' % (precision, recall, f_score) precision, recall, f_score, support, present_labels = precision_recall_fscore_support(y_test, y_pred, average='micro') print 'Micro\t%.4lf\t%.4lf\t%.4lf' % (precision, recall, f_score)
def evaluate_mutiple(ground_truth, prediction, find_max=False, f_beta = 1.0, avg_method=None): """ :param ground_truth: 1-d array, e.g. gt: [1, 1, 2, 2, 3] :param prediction: 1-d array, e.g. prediction: [1, 1, 2, 2, 4] :return: recall, precision, f-value """ prediction_indices = prediction if find_max or len(prediction.shape) == 2: prediction_indices = find_max_indices(prediction) # Find Precision & Recall & F-value precision, recall, f_value, support = None, None, None, None if len(prediction.shape) == 2: M = prediction.shape[1] precision, recall, f_value, support \ = precision_recall_fscore_support(ground_truth, prediction_indices, beta=f_beta, pos_label=M, average=avg_method) else: precision, recall, f_value, support \ = precision_recall_fscore_support(ground_truth, prediction_indices, beta=f_beta, average=avg_method) return precision, recall, f_value
def clf_metrics(p_train, p_test, y_train, y_test): """ Compute metrics on classifier predictions Parameters ---------- p_train : np.array [n_samples] predicted probabilities for training set p_test : np.array [n_samples] predicted probabilities for testing set y_train : np.array [n_samples] Training labels. y_test : np.array [n_samples] Testing labels. Returns ------- clf_scores : dict classifier scores for training set """ y_pred_train = 1*(p_train >= 0.5) y_pred_test = 1*(p_test >= 0.5) train_scores = {} test_scores = {} train_scores['accuracy'] = metrics.accuracy_score(y_train, y_pred_train) test_scores['accuracy'] = metrics.accuracy_score(y_test, y_pred_test) train_scores['mcc'] = metrics.matthews_corrcoef(y_train, y_pred_train) test_scores['mcc'] = metrics.matthews_corrcoef(y_test, y_pred_test) (p, r, f, s) = metrics.precision_recall_fscore_support(y_train, y_pred_train) train_scores['precision'] = p train_scores['recall'] = r train_scores['f1'] = f train_scores['support'] = s (p, r, f, s) = metrics.precision_recall_fscore_support(y_test, y_pred_test) test_scores['precision'] = p test_scores['recall'] = r test_scores['f1'] = f test_scores['support'] = s train_scores['confusion matrix'] = \ metrics.confusion_matrix(y_train, y_pred_train, labels=[0, 1]) test_scores['confusion matrix'] = \ metrics.confusion_matrix(y_test, y_pred_test, labels=[0, 1]) train_scores['auc score'] = \ metrics.roc_auc_score(y_train, p_train + 1, average='weighted') test_scores['auc score'] = \ metrics.roc_auc_score(y_test, p_test + 1, average='weighted') clf_scores = {'train': train_scores, 'test': test_scores} return clf_scores
def melodiness_metrics(m_train, m_test, y_train, y_test): """ Compute metrics on melodiness score Parameters ---------- m_train : np.array [n_samples] melodiness scores for training set m_test : np.array [n_samples] melodiness scores for testing set y_train : np.array [n_samples] Training labels. y_test : np.array [n_samples] Testing labels. Returns ------- melodiness_scores : dict melodiness scores for training set """ m_bin_train = 1*(m_train >= 1) m_bin_test = 1*(m_test >= 1) train_scores = {} test_scores = {} train_scores['accuracy'] = metrics.accuracy_score(y_train, m_bin_train) test_scores['accuracy'] = metrics.accuracy_score(y_test, m_bin_test) train_scores['mcc'] = metrics.matthews_corrcoef(y_train, m_bin_train) test_scores['mcc'] = metrics.matthews_corrcoef(y_test, m_bin_test) (p, r, f, s) = metrics.precision_recall_fscore_support(y_train, m_bin_train) train_scores['precision'] = p train_scores['recall'] = r train_scores['f1'] = f train_scores['support'] = s (p, r, f, s) = metrics.precision_recall_fscore_support(y_test, m_bin_test) test_scores['precision'] = p test_scores['recall'] = r test_scores['f1'] = f test_scores['support'] = s train_scores['confusion matrix'] = \ metrics.confusion_matrix(y_train, m_bin_train, labels=[0, 1]) test_scores['confusion matrix'] = \ metrics.confusion_matrix(y_test, m_bin_test, labels=[0, 1]) train_scores['auc score'] = \ metrics.roc_auc_score(y_train, m_train + 1, average='weighted') test_scores['auc score'] = \ metrics.roc_auc_score(y_test, m_test + 1, average='weighted') melodiness_scores = {'train': train_scores, 'test': test_scores} return melodiness_scores
def eval_models(stream1, stream2, predictor1, predictor2): source1 = multiplex_streams([stream1, stream2], [0.5, 0.5], 1000) source2 = multiplex_streams([stream1, stream2], [0.1, 0.9], 1000) for source in source1, source2: data = source.next() y_est1 = predictor1(data['x']).values()[0].argmax(axis=1) y_est2 = predictor2(data['x']).values()[0].argmax(axis=1) print metrics.precision_recall_fscore_support(data['y'], y_est1) print metrics.precision_recall_fscore_support(data['y'], y_est2)
def metric(self,tag,rank): precision,recall,fbeta,support = precision_recall_fscore_support(self.purchase,tag) print "precision of purchase:",precision print "recall of purchase:",recall PAN,a,b,c = precision_recall_fscore_support(self.rating,rank) print "P @ N :",PAN
def GBDT(x_train,y_train,x_test,y_test): #####GBDT clf = GradientBoostingClassifier() clf.fit(x_train,y_train) predict_y = clf.predict(x_test) auc_score=metrics.roc_auc_score(y_test,predict_y) print 'GBDT auc_score=',auc_score print metrics.precision_recall_fscore_support(y_test,predict_y) return auc_score
def AdaBoost(x_train,y_train,x_test,y_test): #####AdaBoostClassifier clf = AdaBoostClassifier() clf.fit(x_train,y_train) predict_y = clf.predict(x_test) auc_score=metrics.roc_auc_score(y_test,predict_y) print 'AdaBoost auc_score=',auc_score print metrics.precision_recall_fscore_support(y_test,predict_y) return auc_score
def RF(x_train,y_train,x_test,y_test): #####RF clf = RandomForestClassifier() clf.fit(x_train,y_train) predict_y = clf.predict(x_test) auc_score=metrics.roc_auc_score(y_test,predict_y) print 'RF auc_score=',auc_score print metrics.precision_recall_fscore_support(y_test,predict_y) return auc_score
def get_scores(y_pred, y_true): scores = precision_recall_fscore_support(y_true=y_true, y_pred=y_pred, labels=[0,1]) average = precision_recall_fscore_support(y_true=y_true, y_pred=y_pred, average="macro", pos_label=None, labels=[0, 1]) return scores, average
def NBgauss(x_train,y_train,x_test,y_test): ####Naive Bayes (Gaussian likelihood) clf = GaussianNB() clf.fit(x_train,y_train) predict_y = clf.predict(x_test) auc_score=metrics.roc_auc_score(y_test,predict_y) print 'GaussianNB auc_score=',auc_score print metrics.precision_recall_fscore_support(y_test,predict_y) return auc_score
def fmeasure(gl,pl): from sklearn.metrics import precision_recall_fscore_support y_gold=np.array(gl) y_pred=np.array(pl) assert len(y_gold)==len(y_pred) # print y_gold[:10] # print y_pred[:10] prf=precision_recall_fscore_support(y_gold, y_pred, average='macro',pos_label=None) acc=precision_recall_fscore_support(y_gold, y_pred, average='micro',pos_label=None) return prf,acc[0]
def evaluateClassifier(testFolderName,clfFolderName,th=0): clf = Classifier(clfFolderName) batchGen = batchGenerator(50000,testFolderName,'Test') pred = [] exp = [] for _,X,Y in batchGen: pred += clf.predict(X) exp += Y print precision_recall_fscore_support(exp, pred, average='micro') return pred,exp
def evaluate_results(self,expected,predicted): if(len(expected)!=len(predicted)): print "Looks like some values were not predicted properly" print "Precision, Accuracy, Recall, FScore" print precision_recall_fscore_support(expected, predicted,average='binary') cm=confusion_matrix(expected, predicted) print "Confusion matrix" print cm
def apply_Model(temp_data, selectModel): data = {}; data['X_train_ceil'] = temp_data['X_train_ceil']; data['X_test_ceil'] = temp_data['X_test_ceil']; data['y_train_ceil'] = temp_data['y_train_ceil']; data['y_test_ceil'] = temp_data['y_test_ceil']; data['ind_ceil'] = temp_data['ind_ceil'] # feature selection for the floor data['X_train_floor'] = temp_data['X_train_floor']; data['X_test_floor'] = temp_data['X_test_floor']; data['y_train_floor'] = temp_data['y_train_floor'] ; data['y_test_floor'] = temp_data['y_test_floor']; data['ind_floor'] = temp_data['ind_floor']; if(selectModel==1): print "OneVsRest"; classifier_floor = OneVsRestClassifier(LinearSVC(random_state=0)).fit(data['X_train_floor'], data['y_train_floor']) classifier_ceil = OneVsRestClassifier(LinearSVC(random_state=0)).fit(data['X_train_ceil'], data['y_train_ceil']) if(selectModel==2): print "Decision Tree"; classifier_floor = tree.DecisionTreeClassifier().fit(data['X_train_floor'], data['y_train_floor']) classifier_ceil = tree.DecisionTreeClassifier().fit(data['X_train_ceil'], data['y_train_ceil']) if(selectModel==3): print "Nearest Centroid"; classifier_floor = NearestCentroid().fit(data['X_train_floor'], np.ravel(data['y_train_floor'])); classifier_ceil = NearestCentroid().fit(data['X_train_ceil'], np.ravel(data['y_train_ceil'])); if(selectModel==4): print "SGD Classifier"; classifier_floor = SGDClassifier(loss="hinge", penalty="l2").fit(data['X_train_floor'], np.ravel(data['y_train_floor'])); classifier_ceil = SGDClassifier(loss="hinge", penalty="l2").fit(data['X_train_ceil'], np.ravel(data['y_train_ceil'])); train_predict_floor = classifier_floor.predict(data['X_train_floor']); conf_mat_floor = confusion_matrix(train_predict_floor,data['y_train_floor']); train_predict_ceil = classifier_ceil.predict(data['X_train_ceil']); conf_mat_ceil = confusion_matrix(train_predict_ceil, data['y_train_ceil']) y_predict_ceil = classifier_ceil.predict(data['X_test_ceil']) result_ceil = confusion_matrix(y_predict_ceil,data['y_test_ceil']) y_predict_floor = classifier_floor.predict(data['X_test_floor']) result_floor = confusion_matrix(y_predict_floor,data['y_test_floor']) precision_floor, recall_floor, _, _ = precision_recall_fscore_support(data['y_test_floor'], y_predict_floor) precision_ceil, recall_ceil, _, _ = precision_recall_fscore_support(data['y_test_ceil'], y_predict_ceil) acc_ceil = result_ceil.trace()*100/result_ceil.sum(); acc_ceil_train = conf_mat_ceil.trace()*100/conf_mat_ceil.sum(); acc_floor = result_floor.trace()*100/result_floor.sum(); acc_floor_train = conf_mat_floor.trace()*100/conf_mat_floor.sum(); data['acc_ceil_train'] = acc_ceil_train; data['acc_floor_train'] = acc_floor_train; return data, acc_ceil, acc_floor, recall_ceil, recall_floor, result_ceil, result_floor, conf_mat_floor, conf_mat_ceil;
def main(self): dirname = 'features/' rlst = ['radar1','radar2','radar3','radar4'] for rname in rlst: filelist = get_file_list(dirname+rname+'/') trfile = dirname+rname+'/'+rname+'_train.csv' tsfile = dirname+rname+'/'+rname+'_test.csv' tl = False trl = False test = np.array([]) train = np.array([]) for name in filelist: file = dirname+rname+'/'+name if search('p3',name): if tl==False: test = np.genfromtxt(file, delimiter=',') tl = True else: tst = np.genfromtxt(file, delimiter=',') test = np.vstack((test,tst)) # print 'test' # print test.shape else: if trl==False: train = np.genfromtxt(file, delimiter=',') trl = True else: tr = np.genfromtxt(file, delimiter=',') train = np.vstack((train,tr)) # print 'train' # print train.shape np.savetxt(trfile, train, delimiter=",") np.savetxt(tsfile, test, delimiter=",") print rname print '______' clf = Pipeline([ ('feature_selection', LinearSVC(C=0.01, penalty="l1", dual=False)), ('classification', RandomForestClassifier()) ]) X = train[:,0:25] y = train[:,26] # print y clf.fit(X, y) y_ = clf.predict(test[:,0:25]) print y_ # print clf.score(test[:,0:25],test[:,26]) # clf.predict() y = test[:,26] cm = confusion_matrix(y, y_) print cm print cm.shape print clf.score(test[:,0:25],test[:,26]) print precision_recall_fscore_support(y,y_)
def load_digits_small(): # The digits dataset digits = load_digits() # The data that we are interested in is made of 8x8 images of digits, let's # have a look at the first 3 images, stored in the `images` attribute of the # dataset. If we were working from image files, we could load them using # pylab.imread. Note that each image must have the same size. For these # images, we know which digit they represent: it is given in the 'target' of # the dataset. images_and_labels = list(zip(digits.images, digits.target)) for index, (image, label) in enumerate(images_and_labels[:4]): plt.subplot(2, 4, index + 1) plt.axis('off') plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest') plt.title('Training: %i' % label) # To apply a classifier on this data, we need to flatten the image, to # turn the data in a (samples, feature) matrix: n_samples = len(digits.images) data = digits.images.reshape((n_samples, -1)) target = digits.target # only take some classes of digits c1 = 1 c2 = [2] target_bin_index = [i for i, t in enumerate(target) if t == c1 or t in c2] bin_samples = len(target_bin_index) binary_data = data[target_bin_index] binary_target = target[target_bin_index] binary_target[binary_target == c1] = 1 binary_target[binary_target != c1] = -1 nu_gamma_precision = [0, 0, 0] nu_range = [0.1*i for i in range(1,10)] gamma_range = [0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30] for nu in nu_range: for gamma in gamma_range: clf = ocsvm.OCSVM("rbf", nu=nu, gamma=gamma) clf.fit(binary_data[:bin_samples/2]) expected = binary_target[:bin_samples/ 2:] predicted = clf.predict(binary_data[:bin_samples/2:]) precision, recall, f1score, _ = precision_recall_fscore_support(expected, predicted, average='binary') if precision > nu_gamma_precision[2]: nu_gamma_precision = [nu, gamma, precision] print "nu_gamma_precision: %s" % nu_gamma_precision clf = ocsvm.OCSVM("rbf", nu=nu_gamma_precision[0], gamma=nu_gamma_precision[1]) clf.fit(binary_data[:bin_samples/2]) expected = binary_target[:bin_samples/ 2:] predicted = clf.predict(binary_data[:bin_samples/2:]) print("Confusion matrix:\n%s" % confusion_matrix(expected, predicted)) precision, recall, f1score, support = precision_recall_fscore_support(expected, predicted, average='binary') print "precision: %s, recall: %s, f1-score: %s" % (precision, recall, f1score)
def print_analysis(op_csv_list): y_true = [] y_pred = [] for file in op_csv_list: file_csv = pd.read_csv(file) for i, row in enumerate(file_csv.values): y_true.append(row[2]) y_pred.append(row[4]) print confusion_matrix(y_true, y_pred) print precision_recall_fscore_support(y_true, y_pred, average="micro")
def print_analysis(pos_op_csv_lst, neg_op_csv_lst): files = pos_op_csv_lst[:-1] + neg_op_csv_lst[:-1] y_true = [] y_pred = [] for file in files: file_csv = pd.read_csv(file) for i, row in enumerate(file_csv.values): y_true.append(row[2]) y_pred.append(row[4]) print confusion_matrix(y_true, y_pred) print precision_recall_fscore_support(y_true, y_pred, average='micro')
def test_precision_recall_f1_score_multiclass(): """Test Precision Recall and F1 Score for multiclass classification task""" y_true, y_pred, _ = make_prediction(binary=False) # compute scores with default labels introspection p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average=None) assert_array_almost_equal(p, [0.83, 0.33, 0.42], 2) assert_array_almost_equal(r, [0.79, 0.09, 0.90], 2) assert_array_almost_equal(f, [0.81, 0.15, 0.57], 2) assert_array_equal(s, [24, 31, 20]) # averaging tests ps = precision_score(y_true, y_pred, pos_label=1, average='micro') assert_array_almost_equal(ps, 0.53, 2) rs = recall_score(y_true, y_pred, average='micro') assert_array_almost_equal(rs, 0.53, 2) fs = f1_score(y_true, y_pred, average='micro') assert_array_almost_equal(fs, 0.53, 2) ps = precision_score(y_true, y_pred, average='macro') assert_array_almost_equal(ps, 0.53, 2) rs = recall_score(y_true, y_pred, average='macro') assert_array_almost_equal(rs, 0.60, 2) fs = f1_score(y_true, y_pred, average='macro') assert_array_almost_equal(fs, 0.51, 2) ps = precision_score(y_true, y_pred, average='weighted') assert_array_almost_equal(ps, 0.51, 2) rs = recall_score(y_true, y_pred, average='weighted') assert_array_almost_equal(rs, 0.53, 2) fs = f1_score(y_true, y_pred, average='weighted') assert_array_almost_equal(fs, 0.47, 2) assert_raises(ValueError, precision_score, y_true, y_pred, average="samples") assert_raises(ValueError, recall_score, y_true, y_pred, average="samples") assert_raises(ValueError, f1_score, y_true, y_pred, average="samples") assert_raises(ValueError, fbeta_score, y_true, y_pred, average="samples", beta=0.5) # same prediction but with and explicit label ordering p, r, f, s = precision_recall_fscore_support( y_true, y_pred, labels=[0, 2, 1], average=None) assert_array_almost_equal(p, [0.83, 0.41, 0.33], 2) assert_array_almost_equal(r, [0.79, 0.90, 0.10], 2) assert_array_almost_equal(f, [0.81, 0.57, 0.15], 2) assert_array_equal(s, [24, 20, 31])
def compareResult(result): result.sort() data = [] with open(data3result,'rb') as csvfile: for line in csvfile: if len(line.strip()) > 0: data.append(int(line) - 1) data = np.array(data) # print data # print result print precision_recall_fscore_support(data, result, average='micro')
plt.savefig('/Users/Mehraveh/Desktop/class_probs_leave_one_out.png', dpi=120) sorted_ptms = sorted(range(len(imp)), key=lambda k: imp[k], reverse=True) sorted_ptms1 = [x + 1 for x in sorted_ptms] final = pd.DataFrame(sorted_ptms, label[sorted_ptms1]) plt.figure(1, figsize=(6, 6)) plt.figure(figsize=(12, 8)) xx = range(n_features) plt.bar(xx, imp[sorted_ptms], 1, color="blue") plt.xticks(xx, label[sorted_ptms1], rotation='vertical') plt.savefig('/Users/Mehraveh/Desktop/impFeatures.png', dpi=300) predicted = y_pred y_test = y precision, recall, fscore, support = precision_recall_fscore_support( y_test, predicted) ### What are the three unknown classes? X_test = data.ix[data[0] == 0, :] X_test = X_test.drop(0, 1) X_test = np.asmatrix(X_test) y_test = data[0] y_test = y_test.ix[data[0] == 0] y = y.ravel() X_train = X y_train = y clf.fit(X_train, y_train) y_pred = clf.predict(X_test) class_probs_unknowns = clf.predict_proba(X_test) prob_loo_unknowns = class_probs_unknowns
def train(): model = ResNet().to(device) # optimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE, amsgrad=False, weight_decay=0.0005) # scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 5, gamma=0.5) optimizer = torch.optim.SGD(model.parameters(), lr=LEARNING_RATE, momentum=0.9) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 10, gamma=0.5) total_step = len(train_loader) plot_x = [] plot_acc = [] plot_recall = [] plot_precision = [] plot_f1 = [] plot_x_loss = [] plot_loss = [] for epoch in range(EPOCH): torch.cuda.empty_cache() for step, (images, labels) in enumerate(train_loader): images = images.to(device) labels = labels.to(device) outputs = model(images) loss = loss_func(outputs, labels) optimizer.zero_grad() loss.backward() optimizer.step() if step % 20 == 0: print('Epoch [{}/{}], Step [{}/{}], Loss: {:.7f}'.format( epoch + 1, EPOCH, step + 1, total_step, loss.item())) if step % 50 == 0: plot_x_loss.append(step / 235 + epoch) plot_loss.append(loss.item()) scheduler.step() true_target = torch.FloatTensor() pre_target = torch.FloatTensor() correct_vali = 0 total_vali = 0 for images_vali, target in vali_loader: images_vali = images_vali.to(device) target = target.to(device) with torch.no_grad(): pred = model(images_vali) maxk = max((1, 5)) target_resize = target.view(-1, 1) _, predicted = pred.topk(maxk, 1, True, True) loss = loss_func(pred, target) correct_vali += torch.eq(predicted, target_resize).sum().item() total_vali += target.size(0) temp = predicted.narrow(1, 0, 1) temp = torch.squeeze(temp, 1) temp = temp.cpu() temp = temp.view(-1).float() pre_target = torch.cat((pre_target, temp), 0) temp = target.cpu() temp = temp.view(-1).float() true_target = torch.cat((true_target, temp), 0) score_precision, score_recall, score_f1, _ = precision_recall_fscore_support( true_target, pre_target, average='macro') plot_x.append(epoch + 1) plot_acc.append(correct_vali / total_vali) plot_f1.append(score_f1) plot_precision.append(score_precision) plot_recall.append(score_recall) print( 'Epoch {} Accuracy {:.4f} %, loss {:.4f}\nf1-score {:.4f} recall {:.4f} precision {:.4f}' .format(epoch + 1, 100 * correct_vali / total_vali, loss.item(), score_f1, score_recall, score_precision)) torch.save(model.state_dict(), './model&img/model.pt') plotScore(plot_x, plot_acc, plot_recall, plot_precision, plot_f1, plot_x_loss, plot_loss)
y, test_size=0.25, random_state=0) # 2. Scaling from sklearn.preprocessing import StandardScaler scaler_X = StandardScaler() X_train = scaler_X.fit_transform(X_train) X_test = scaler_X.transform(X_test) # Machine: Classifier | NB: Gaussian Naive Bayes from sklearn.naive_bayes import GaussianNB classifier = GaussianNB() classifier.fit(X_train, y_train) # Predictions y_pred = classifier.predict(X_test) # Validating Predictions using Confusion Matrix from sklearn.metrics import confusion_matrix cm = confusion_matrix(y_test, y_pred) print(cm) from sklearn.metrics import precision_recall_fscore_support prf = precision_recall_fscore_support(y_test, y_pred) print('\t\t\t\t ZERO\t\t\tONE') print('Precision\t:', prf[0] * 100) print('Recall\t\t:', prf[1] * 100) print('F1 Measure\t:', prf[2] * 100) print('Support\t\t:', prf[3])
def train(self, train_indices=None, test_indices=None): #Read data if self.read_data_batch: self.read_rgbd_data_batch(self.read_data_ratio) elif train_indices is not None: self.read_grasp_data_from_indices(train_indices, test_indices) else: self.read_rgbd_data() #Create the variables. self.create_net_var() #Call the network building function. self.cost_function() #Initialize the variables init = tf.global_variables_initializer() saver = tf.train.Saver() tf.get_default_graph().finalize() logs_path = self.logs_path + '_train' # op to write logs to Tensorboard summary_writer = tf.summary.FileWriter(logs_path, graph=tf.get_default_graph()) start_time = time.time() train_costs = [] train_pred_errors = [] test_pred_errors = [] learn_rate = 0.001 # Launch the graph with tf.Session() as sess: sess.run(init) # Training cycle for epoch in range(self.training_epochs): if epoch % self.rate_dec_epochs == 0 and epoch != 0: learn_rate *= 0.1 if self.read_data_batch and epoch % self.read_batch_epochs == 0 and epoch != 0: self.read_rgbd_data_batch(self.read_data_ratio) #[pool_1_out, fc_config_out] = sess.run([self.max_pool_1, self.fc_config], feed_dict=self.feed_dict_func(True, learn_rate)) #print np.array(pool_1_out).shape, np.array(fc_config_out).shape [_, cost_output, pred_error_output, train_summary_output, f2vis_train_output, labels_train_output, pred_train_output] = \ sess.run([self.optimizer, self.cost, self.pred_error, self.train_summary, self.feature_to_vis, self.holder_labels, self.pred], feed_dict=self.feed_dict_func(True, learn_rate)) # Write logs at every iteration summary_writer.add_summary(train_summary_output, epoch) # Display logs per epoch step train_costs.append(cost_output) train_pred_errors.append(pred_error_output) [ pred_error_test_output, pred_err_test_sum_output, f2vis_output_test, labels_test_output, pred_test_output ] = sess.run([ self.pred_error_test, self.pred_err_test_sum, self.feature_to_vis, self.holder_labels, self.pred ], feed_dict=self.feed_dict_func(False)) test_pred_errors.append(pred_error_test_output) summary_writer.add_summary(pred_err_test_sum_output, epoch) if epoch % self.display_step == 0: print 'epoch: ', epoch print 'labels_train_output:', labels_train_output print 'pred_train_output', pred_train_output print 'labels_test_output:', labels_test_output print 'pred_test_output', pred_test_output print 'train_cost: ', cost_output print 'pred_error_train: ', pred_error_output print 'train pred std: ', np.std(pred_train_output[:, 0]) print 'pred_error_test: ', pred_error_test_output print 'test pred std: ', np.std(pred_test_output[:, 0]) pred_train_output[pred_train_output > 0.5] = 1. pred_train_output[pred_train_output <= 0.5] = 0. train_prfc = precision_recall_fscore_support( labels_train_output, pred_train_output) print 'training precision, recall, fscore, support:' print train_prfc pred_test_output[pred_test_output > 0.5] = 1. pred_test_output[pred_test_output <= 0.5] = 0. test_prfc = precision_recall_fscore_support( labels_test_output, pred_test_output) print 'testing precision, recall, fscore, support:' print test_prfc if epoch % (10 * self.display_step) == 0: tsne_vis.tsne_vis( f2vis_train_output, labels_train_output[:, 0], '../models/tsne/train_tsne_' + str(epoch) + '_gt.png') tsne_vis.tsne_vis( f2vis_output_test, labels_test_output[:, 0], '../models/tsne/test_tsne_' + str(epoch) + '_gt.png') tsne_vis.tsne_vis( f2vis_train_output, pred_train_output[:, 0], '../models/tsne/train_tsne_' + str(epoch) + '_pred.png') tsne_vis.tsne_vis( f2vis_output_test, pred_test_output[:, 0], '../models/tsne/test_tsne_' + str(epoch) + '_pred.png') print("Optimization Finished!") saver.save(sess, self.cnn_model_path) fig, ax = plt.subplots(figsize=(8, 8)) ax.plot(np.linspace(1, self.training_epochs, self.training_epochs), train_costs, 'r') fig.savefig('../models/train_costs.png') plt.clf() plt.close(fig) fig, ax = plt.subplots(figsize=(8, 8)) ax.plot(np.linspace(1, self.training_epochs, self.training_epochs), train_pred_errors, 'b') fig.savefig('../models/train_pred_errors.png') plt.clf() plt.close(fig) fig, ax = plt.subplots(figsize=(8, 8)) ax.plot(np.linspace(1, self.training_epochs, self.training_epochs), test_pred_errors, 'k') fig.savefig('../models/test_pred_errors.png') plt.clf() plt.close(fig) elapsed_time = time.time() - start_time print 'Total training elapsed_time: ', elapsed_time
if FLAGS.early_stopping > 0 and epoch > FLAGS.early_stopping and cost_val[-1] > np.mean(cost_val[-(FLAGS.early_stopping+1):-1]): print("Early stopping...") break print("Optimization Finished!") # Best results print('Best epoch:', best_epoch) print("Test set results:", "cost=", "{:.5f}".format(best_cost), "accuracy=", "{:.5f}".format(best_acc)) print("Test Precision, Recall and F1-Score...") print(metrics.classification_report(labels, preds, digits=4)) print("Macro average Test Precision, Recall and F1-Score...") print(metrics.precision_recall_fscore_support(labels, preds, average='macro')) print("Micro average Test Precision, Recall and F1-Score...") print(metrics.precision_recall_fscore_support(labels, preds, average='micro')) ''' # For visualization doc_vectors = [] for i in range(len(test_doc_embeddings)): doc_vector = test_doc_embeddings[i] doc_vector_str = ' '.join([str(x) for x in doc_vector]) doc_vectors.append(str(np.argmax(test_y[i])) + ' ' + doc_vector_str) doc_embeddings_str = '\n'.join(doc_vectors) with open('data/' + FLAGS.dataset + '_doc_vectors.txt', 'w'): f.write(doc_embeddings_str) '''
c_com_weight, early_type, conf_threshold_1_vs_2 ]) writer.writerow([ dataset_name, method, "cost_avg_baseline", cost_baseline / len(dt_final), c_miss_weight, c_action_weight, c_postpone_weight, c_com_weight, early_type, conf_threshold_1_vs_2 ]) dt_final.prediction = dt_final.prediction.replace(2, 1) #print(dt_final.prediction) # calculate precision, recall etc. prec, rec, fscore, _ = precision_recall_fscore_support( dt_final.actual, dt_final.prediction, pos_label=1, average="binary") tn, fp, fn, tp = confusion_matrix(dt_final.actual, dt_final.prediction).ravel() # calculate earliness based on the "true alarms" only tmp = dt_final[(dt_final.prediction == 1) & (dt_final.actual == 1)] earliness = (1 - ((tmp.prefix_nr - 1) / tmp.case_length)) tmp = dt_final[(dt_final.prediction == 1)] earliness_alarms = (1 - ((tmp.prefix_nr - 1) / tmp.case_length)) writer.writerow([ dataset_name, method, "fire_delay", myopic_param,
def cifar100(config, path, param): print("START CIFAR100") use_cuda = config.general.use_cuda and torch.cuda.is_available() torch.manual_seed(config.general.seed) device = torch.device("cuda" if use_cuda else "cpu") print("START TRAINING TARGET MODEL") data_train_target = custum_CIFAR100(True, 0, config, '../data', train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize( (0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ])) data_test_target = custum_CIFAR100(True, 0, config, '../data', train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize( (0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ])) criterion = nn.CrossEntropyLoss() train_loader_target = torch.utils.data.DataLoader( data_train_target, batch_size=config.learning.batch_size, shuffle=True) test_loader_target = torch.utils.data.DataLoader( data_test_target, batch_size=config.learning.batch_size, shuffle=True) dataloaders_target = { "train": train_loader_target, "val": test_loader_target } dataset_sizes_target = { "train": len(data_train_target), "val": len(data_test_target) } model_target = Net_cifar100().to(device) optimizer = optim.SGD(model_target.parameters(), lr=config.learning.learning_rate, momentum=config.learning.momentum) exp_lr_scheduler = lr_scheduler.StepLR( optimizer, step_size=config.learning.decrease_lr_factor, gamma=config.learning.decrease_lr_every) model_target, best_acc_target, data_test_set, label_test_set, class_test_set = train_model( model_target, criterion, optimizer, exp_lr_scheduler, dataloaders_target, dataset_sizes_target, num_epochs=config.learning.epochs) np.savetxt(path + "/res_train_target_" + str(param) + ".csv", best_acc_target) plot_title = 'CIFAR100' drawLossAcc(best_acc_target, plot_title, path) print("START TRAINING SHADOW MODEL") all_shadow_models = [] all_dataloaders_shadow = [] data_train_set = [] label_train_set = [] class_train_set = [] for num_model_sahdow in range(config.general.number_shadow_model): criterion = nn.CrossEntropyLoss() data_train_shadow = custum_CIFAR100(False, num_model_sahdow, config, '../data', train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize( (0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ])) data_test_shadow = custum_CIFAR100(False, num_model_sahdow, config, '../data', train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize( (0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ])) train_loader_shadow = torch.utils.data.DataLoader( data_train_shadow, batch_size=config.learning.batch_size, shuffle=True) test_loader_shadow = torch.utils.data.DataLoader( data_test_shadow, batch_size=config.learning.batch_size, shuffle=True) dataloaders_shadow = { "train": train_loader_shadow, "val": test_loader_shadow } dataset_sizes_shadow = { "train": len(data_train_shadow), "val": len(data_test_shadow) } model_shadow = Net_cifar100().to(device) optimizer = optim.SGD(model_shadow.parameters(), lr=config.learning.learning_rate, momentum=config.learning.momentum) exp_lr_scheduler = lr_scheduler.StepLR( optimizer, step_size=config.learning.decrease_lr_factor, gamma=config.learning.decrease_lr_every) model_shadow, best_acc_sh, data_train_set_unit, label_train_set_unit, class_train_set_unit = train_model( model_shadow, criterion, optimizer, exp_lr_scheduler, dataloaders_target, dataset_sizes_target, num_epochs=config.learning.epochs) data_train_set.append(data_train_set_unit) label_train_set.append(label_train_set_unit) class_train_set.append(class_train_set_unit) np.savetxt( path + "/res_train_shadow_" + str(num_model_sahdow) + "_" + str(param) + ".csv", best_acc_sh) all_shadow_models.append(model_shadow) all_dataloaders_shadow.append(dataloaders_shadow) print("START GETTING DATASET ATTACK MODEL") data_train_set = np.concatenate(data_train_set) label_train_set = np.concatenate(label_train_set) class_train_set = np.concatenate(class_train_set) #data_test_set, label_test_set, class_test_set = get_data_for_final_eval([model_target], [dataloaders_target], device) #data_train_set, label_train_set, class_train_set = get_data_for_final_eval(all_shadow_models, all_dataloaders_shadow, device) data_train_set, label_train_set, class_train_set = shuffle( data_train_set, label_train_set, class_train_set, random_state=config.general.seed) data_test_set, label_test_set, class_test_set = shuffle( data_test_set, label_test_set, class_test_set, random_state=config.general.seed) #print(data_train_set.shape, data_test_set.shape) print("dataset train size", len(label_train_set)) print("dataset test size", len(label_test_set)) print("START FITTING ATTACK MODEL") model = lgb.LGBMClassifier(objective='binary', reg_lambda=config.learning.ml.reg_lambd, n_estimators=config.learning.ml.n_estimators) model.fit(data_train_set, label_train_set) y_pred_lgbm = model.predict(data_test_set) precision_general, recall_general, _, _ = precision_recall_fscore_support( y_pred=y_pred_lgbm, y_true=label_test_set, average="macro") accuracy_general = accuracy_score(y_true=label_test_set, y_pred=y_pred_lgbm) precision_per_class, recall_per_class, accuracy_per_class = [], [], [] for idx_class, classe in enumerate(data_train_target.classes): all_index_class = np.where(class_test_set == idx_class) precision, recall, _, _ = precision_recall_fscore_support( y_pred=y_pred_lgbm[all_index_class], y_true=label_test_set[all_index_class], average="macro") accuracy = accuracy_score(y_true=label_test_set[all_index_class], y_pred=y_pred_lgbm[all_index_class]) precision_per_class.append(precision) recall_per_class.append(recall) accuracy_per_class.append(accuracy) print("END CIFAR100") return (precision_general, recall_general, accuracy_general, precision_per_class, recall_per_class, accuracy_per_class)
def run_classifer(X_train, s_train, y_train, X_test, s_test, y_test): s_train = np.array(s_train) # samples x features s_test = np.array(s_test) num_labels = 15 batch_size = 100 stemmer = sb.SnowballStemmer('english') swlist = sw.words('english') swlist += [stemmer.stem(w) for w in swlist] swlist += [ "'d", "'s", 'abov', 'ani', 'becaus', 'befor', 'could', 'doe', 'dure', 'might', 'must', "n't", 'need', 'onc', 'onli', 'ourselv', 'sha', 'themselv', 'veri', 'whi', 'wo', 'would', 'yourselv' ] #complained about not having these as stop words pubs = [ 'buzzfe', 'buzzf', 'npr', 'cnn', 'vox', 'reuter', 'breitbart', 'fox', 'guardian', 'review', 'theatlant' ] punct = [ ] #[':', '..', '“', '@', '%', ';', '→', ')', '#', '(', '*', '&', '[', ']', '…', '?','—', '‘', '$'] #gonna leave these in for now swlist += pubs swlist += punct if sys.argv[4].lower() == 'true': tkzr = StemTokenizer() else: tkzr = None if sys.argv[5].lower() != 'true': swlist = [] #what features are we using? if sys.argv[7].lower() == 'word': count_vect = CountVectorizer(stop_words=swlist, tokenizer=tkzr) count_vect.fit(X_train) X_train = count_vect.transform(X_train) X_test = count_vect.transform(X_test) tfidf_transformer = TfidfTransformer() tfidf_transformer.fit(X_train) X_train = tfidf_transformer.transform(X_train) X_test = tfidf_transformer.transform(X_test) elif sys.argv[7].lower() == 'topic': count_vect = CountVectorizer(stop_words=swlist, tokenizer=tkzr) count_vect.fit(X_train) X_train = count_vect.transform(X_train) X_test = count_vect.transform(X_test) lda_model = LatentDirichletAllocation(n_components=10) lda_model.fit(X_train) X_train = lda_model.transform(X_train) X_test = lda_model.transform(X_test) elif sys.argv[7].lower() == 'style': X_train = csr_matrix(s_train) X_test = csr_matrix(s_test) elif sys.argv[7].lower() == 'all': count_vect = CountVectorizer(stop_words=swlist, tokenizer=tkzr) count_vect.fit(X_train) X_train = count_vect.transform(X_train) X_test = count_vect.transform(X_test) tfidf_transformer = TfidfTransformer() tfidf_transformer.fit(X_train) X_train_tf = tfidf_transformer.transform(X_train) X_test_tf = tfidf_transformer.transform(X_test) print(type(X_train_tf)) lda_model = LatentDirichletAllocation(n_components=10) lda_model.fit(X_train) X_train_lda = lda_model.transform(X_train) X_test_lda = lda_model.transform(X_test) print(type(X_train_lda)) X_train = csr_matrix( sparse.hstack( [X_train_tf, csr_matrix(X_train_lda), csr_matrix(s_train)])) X_test = csr_matrix( sparse.hstack( [X_test_tf, csr_matrix(X_test_lda), csr_matrix(s_test)])) print(type(X_train)) # sparse.save_npz("X_train" + sys.argv[6] + ".npz", X_train) # sparse.save_npz("X_test" + sys.argv[6] + ".npz", X_test) else: sys.exit('unknown features') encoder = LabelBinarizer() encoder.fit(y_train) y_train = encoder.transform(y_train) y_test = encoder.transform(y_test) # np.save('X_train.npy', X_train) # np.save('X_test.npy', X_test) # np.save('y_train.npy', y_train) # np.save('y_test.npy', y_test) # sparse.save_npz("y_train" + sys.argv[6] + ".npz", y_train) # sparse.save_npz("y_test" + sys.argv[6] + ".npz", y_test) # load everything back # X_train = sparse.load_npz("X_train.npz") input_dim = X_train.shape[1] model = Sequential() model.add(Dense(512, input_shape=(input_dim, ))) model.add(Activation('relu')) model.add(Dropout(0.3)) model.add(Dense(512)) model.add(Activation('relu')) model.add(Dropout(0.3)) model.add(Dense(num_labels)) model.add(Activation('softmax')) model.summary() model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) history = model.fit(X_train, y_train, batch_size=batch_size, epochs=5, verbose=1, validation_split=0.1) # model.model.save(sys.argv[6] + '.h5') # X_train = np.load('X_train.npy') # X_test = np.load('X_test.npy') # y_train = np.load('y_train.npy') # y_test = np.load('y_test.npy') # model = keras.models.load_model(sys.argv[6] + '.h5') score = model.evaluate(X_test, y_test, batch_size=batch_size, verbose=1) print('Test accuracy:', score[1]) y_pred = model.predict(X_test, batch_size=batch_size, verbose=1) predicted = np.argmax(y_pred, axis=1) p, r, fs, s = precision_recall_fscore_support(np.argmax(y_test, axis=1), predicted) print(p, r, fs, s)
if (acc > max_acc): cl_model.saver.save(cl_model.sess, cl_save_path + 'cl_model.ckpt', global_step=epoch_count) max_acc = acc # Create error case # print("Create error case") test_avg_loss, predict_set, true_set, train_prob = cl_model.predict( train_batch_x, train_batch_y) # print("Train true_set_len: ", len(true_set)) # print(train_prob[:10]) assert len(true_set) == len(train_prob) acc = accuracy_score(true_set, predict_set) performance = metrics.precision_recall_fscore_support(true_set, predict_set, average='binary') precision = performance[0] recall = performance[1] f1 = performance[2] print( "Train test loss: {} accuracy: {} precision: {} recall: {} f1: {}". format(test_avg_loss, round(acc, 6), round(precision, 6), round(recall, 6), round(f1, 6))) t1p0 = 0 t1p0_list = [] t1p0_prob = [] t0p1 = 0 t0p1_list = []
def train_10_fold_balanced(dv, tags): Xl, Xr, y = load_dataset(dv, tags) Xl, Xr, y = shuffle(Xl, Xr, y, random_state=0) skf = StratifiedKFold(n_splits=10) avg_accuracy = 0. avg_recall = 0. avg_precision = 0. avg_f1_score = 0. fold_index = 0 for train_idx, test_idx in skf.split(Xl, y): t_beg = time.clock() print ('*' * 40 + str(fold_index) + '*' * 40) fold_path = os.path.join(vector_name, str(fold_index)) if os.path.exists(vector_name) is not True: os.mkdir(vector_name) if os.path.exists(fold_path) is not True: os.mkdir(fold_path) train_X_left = Xl[train_idx] train_X_right = Xr[train_idx] train_Y = y[train_idx] train_X_left, train_X_right, train_Y = shuffle(train_X_left, train_X_right, train_Y, random_state=0) test_X_left = Xl[test_idx] test_X_right = Xr[test_idx] test_Y = y[test_idx] validate_X_left = test_X_left[:256] validate_X_right = test_X_right[:256] validate_Y = test_Y[:256] X_left = Input(shape=(bin_vec_dim, )) X_right = Input(shape=(bin_vec_dim, )) predictions = classification(X_left, X_right) model = Model(inputs=[X_left, X_right], outputs=predictions) model.compile(optimizer=K.optimizers.adam(lr=0.001), loss=K.losses.binary_crossentropy, metrics=['accuracy']) model.fit([train_X_left, train_X_right], train_Y, epochs=4,verbose=1, batch_size=batch_size) t_end = time.clock() print('Time cost: %.2f' % (t_end - t_beg)) model.save(filepath=os.path.join(fold_path, 'model.ckpt')) y_pred = model.predict([test_X_left, test_X_right], verbose=1, batch_size=batch_size) y_pred = np.round(y_pred) accuracy = accuracy_score(test_Y, y_pred) precision, recall, fscore, _ = precision_recall_fscore_support(test_Y, y_pred, average='binary') print("accuracy: %.4f, recall: %.4f, " "precision: %.4f, f1 score: %.4f\n" % ( accuracy, recall, precision, fscore)) fout.write('*' * 80 + '\n') fout.write('Fold %d:\n' % (fold_index)) fout.write('Time cost: %.2f\n' % (t_end - t_beg)) fout.write("Fold index: %d, accuracy: %.4f, recall: %.4f, " "precision: %.4f, f1 score: %.4f\n" % ( fold_index, accuracy, recall, precision, fscore)) fout.flush() avg_accuracy += accuracy avg_precision += precision avg_recall += recall avg_f1_score += fscore print('*' * 80) fold_index += 1 avg_accuracy /= 10.0 avg_precision /= 10.0 avg_recall /= 10.0 avg_f1_score /= 10.0 print('Avg accuracy: %.4f, avg recall: %.4f, avg precision: %.4f, avg f1 ' 'score: %.4f' % ( avg_accuracy, avg_recall, avg_precision, avg_f1_score)) fout.write('*' * 80 + '\n') fout.write( 'Avg accuracy: %.4f, avg recall: %.4f, avg precision: %.4f, avg f1 ' 'score: %.4f\n' % (avg_accuracy, avg_recall, avg_precision, avg_f1_score))
# View The Accuracy Of Our Limited Feature (2 Features) Model print("Accuracy of limited features : ", end=" ") print(accuracy_score(ytest, y_important_pred)) print() print() ##print("training accuracy: {}".format(100*clf.score(xtrain,ytrain))) ##print("testing accuracy: {}".format(100*clf.score(xtest,ytest))) y_true = ytest y_pred = clf.predict(xtest) from sklearn.metrics import precision_recall_fscore_support, accuracy_score accuracy = accuracy_score(y_true, y_pred) precision, recall, f1_score, _ = precision_recall_fscore_support( y_true, y_pred, average='binary') print("For full feature dataset...") print() print("Accuracy: ", accuracy) print("Precision: ", precision) print("Recall: ", recall) print("F1 score: ", f1_score) print() print() accuracy_imp = accuracy_score(y_true, y_important_pred) precision_imp, recall_imp, f1_score_imp, _ = precision_recall_fscore_support( y_true, y_important_pred, average='binary') print("For limites feature dataset...")
def scores(key, paths, config): import mapreduce print(key) values = [mapreduce.OutputCollector(p) for p in paths] values = [item.load() for item in values] y_true = [item["y_true"].ravel() for item in values] y_pred = [item["y_pred"].ravel() for item in values] y_true = np.concatenate(y_true) y_pred = np.concatenate(y_pred) prob_pred = [item["proba_pred"].ravel() for item in values] prob_pred = np.concatenate(prob_pred) p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average=None) auc = roc_auc_score(y_true, prob_pred) #area under curve score. betas = np.hstack([item["beta"] for item in values]).T # threshold betas to compute fleiss_kappa and DICE betas_t = np.vstack([ array_utils.arr_threshold_from_norm2_ratio(betas[i, :], .99)[0] for i in range(betas.shape[0]) ]) #Compute pvalue success = r * s success = success.astype('int') prob_class1 = np.count_nonzero(y_true) / float(len(y_true)) pvalue_recall0_true_prob = binom_test(success[0], s[0], 1 - prob_class1, alternative='greater') pvalue_recall1_true_prob = binom_test(success[1], s[1], prob_class1, alternative='greater') pvalue_recall0_unknwon_prob = binom_test(success[0], s[0], 0.5, alternative='greater') pvalue_recall1_unknown_prob = binom_test(success[1], s[1], 0.5, alternative='greater') pvalue_recall_mean = binom_test(success[0] + success[1], s[0] + s[1], p=0.5, alternative='greater') scores = OrderedDict() try: a, l1, l2, tv = [float(par) for par in key.split("_")] scores['a'] = a scores['l1'] = l1 scores['l2'] = l2 scores['tv'] = tv left = float(1 - tv) if left == 0: left = 1. scores['l1_ratio'] = float(l1) / left except: pass scores['recall_0'] = r[0] scores['recall_1'] = r[1] scores['recall_mean'] = r.mean() scores["auc"] = auc scores['pvalue_recall0_true_prob_one_sided'] = pvalue_recall0_true_prob scores['pvalue_recall1_true_prob_one_sided'] = pvalue_recall1_true_prob scores[ 'pvalue_recall0_unknwon_prob_one_sided'] = pvalue_recall0_unknwon_prob scores[ 'pvalue_recall1_unknown_prob_one_sided'] = pvalue_recall1_unknown_prob scores['pvalue_recall_mean'] = pvalue_recall_mean scores['prop_non_zeros_mean'] = float(np.count_nonzero(betas_t)) / \ float(np.prod(betas.shape)) scores['param_key'] = key return scores
X = (X - np.min(X, 0)) / (np.max(X, 0) + 0.0001) # 0-1 scaling classifier = joblib.load(PATH + '/data/' + sys.argv[1] + '/pkl/' + sys.argv[1] + '_classifier.pkl') logistic_classifier = joblib.load(PATH + '/data/' + sys.argv[1] + '/pkl/' + sys.argv[1] + '_logistic_classifier.pkl') CP = classifier.predict(X) LCP = logistic_classifier.predict(X) retCP = [] retLCP = [] for i in range(len(CP)): retCP.append(reverse_dic[CP[i]]) retLCP.append(reverse_dic[LCP[i]]) cps, crs, cfs, cs = metrics.precision_recall_fscore_support( CP, Y, average='weighted') lcps, lcrs, lcfs, lcs = metrics.precision_recall_fscore_support( LCP, Y, average='weighted') test = open( PATH + '/data/' + sys.argv[1] + '/test/' + sys.argv[2] + '.test', 'w') json.dump( { 'ntb': { 'samples': samples, 'score': classifier.score(X, Y), 'classifier_recall_score': crs, 'classifier_precision_score': cps, 'classifier_f1_score': cfs, 'logistic_classifier_recall_score': lcrs,
print(df_result) # Making the Confusion Matrix from sklearn.metrics import confusion_matrix,accuracy_score, precision_score from sklearn.metrics import precision_recall_fscore_support cm = confusion_matrix(y, y_pred_all) print(cm) print(cm[1][1]) #accuracy -number of instance correctly classified acsc = accuracy_score(y, y_pred_all) print("Accuracy:") print(acsc) #precision, recall, fscore, support precision, recall, fscore, support = precision_recall_fscore_support(y, y_pred_all,average='weighted') df_metrics = pd.DataFrame([[acsc, precision, recall, fscore]], index=[0], columns=['accuracy','precision', 'recall', 'fscore']) print(df_metrics) print("precision:") print(precision) print("recall:") print(recall) print("fscore:") print(fscore)
combined_training_set_target = combined_training_set.iloc[:, -1] testing_set_features = data_test.iloc[:, :-1] testing_set_target = data_test.iloc[:, -1] # Define the classifier used classifier = GaussianNB() # Fit accordingly to our combined training and validation set classifier.fit(combined_training_set_features, combined_training_set_target) # Finally, predict the test set the best estimator that GridSearch has scored during training test_prediction = classifier.predict(testing_set_features) # Metrics test_confusion = confusion_matrix(testing_set_target, test_prediction) p1, r1, f1, _ = precision_recall_fscore_support(testing_set_target, test_prediction, zero_division='warn') p2, r2, f2, _ = precision_recall_fscore_support(testing_set_target, test_prediction, average='weighted') p3, r3, f3, _ = precision_recall_fscore_support(testing_set_target, test_prediction, average='macro') test_accuracy = accuracy_score(testing_set_target, test_prediction) # Write results and metrics to file file = open(output_results_file_path, 'w', encoding='utf8') writer = csv.writer(file, quotechar='"', quoting=csv.QUOTE_ALL, lineterminator='\n') count = 1 for x in test_prediction: writer.writerow([count, x]) count += 1 writer.writerow("") writer.writerow("") writer.writerow(["Confusion Matrix"]) letter_header = letters.copy()
for x in args.FEAT: if x not in feats_sizes: raise Exception("Unidentified feat: {}".format(x)) folddir = args.folddir outf = args.outf model_name = args.modelname max_len = args.max_len epochs = args.epochs batch_size = args.batch_size FEAT = args.FEAT VOCAB_SIZE = sum(map(feats_sizes.__getitem__, FEAT)) model = create_model() preds = train_and_evaluate_model(model, data_train_gen=Generator( f"{folddir}/train", get_feats), data_eval_folder=f"{folddir}/eval", data_test_folder=f"{folddir}/test", model_name=model_name, epochs=epochs) y_test, y_pred = preds print(",".join(FEAT)) p, r, f1, _ = precision_recall_fscore_support(y_test, y_pred, average='macro') print(" ==== Results ==== ") print(f"{p}\t{r}\t{f1}") print("=" * 20) np.save(outf, preds)
loss, accuracy = model.evaluate( { 'claims': test_claims, 'sentences': test_sents }, test_model.test_data["label"]) print("test loss ", loss) print("test accuracy ", accuracy) batch_size = 32 y_pred = (np.asarray( model.predict({ 'claims': test_claims, 'sentences': test_sents }, batch_size=batch_size))).round() print( "score of lstm ", precision_recall_fscore_support(test_model.test_data["label"], y_pred, average="binary")) f = open("lstm_sent_ret_results.txt", "w") f.write("accuracy " + str(accuracy)) f.write("loss ", +str(loss)) f.write( "precision recall f1 ", +str( precision_recall_fscore_support( test_model.test_data["label"], y_pred, average="binary"))) f.close()
def metric_evaluate(y_true, y_pred, model_name): F1_score = f1_score(y_true, y_pred, average='macro') acc = accuracy_score(y_true=y_true, y_pred=y_pred) recall_score_ = recall_score(y_true, y_pred, average='macro') cohen_kappa_score_ = cohen_kappa_score(y_true, y_pred) confmat = confusion_matrix(y_true=y_true, y_pred=y_pred) clf_report = classification_report(y_true, y_pred) report = np.array(precision_recall_fscore_support(y_true, y_pred)) class1_report = report[:, 1] class1_metric = list(class1_report[:-1]) metrics = [] def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues): """ This function prints and plots the confusion matrix. Normalization can be applied by setting `normalize=True`. """ import itertools if normalize: cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] print("Normalized confusion matrix") else: print('Confusion matrix, without normalization') print(cm) plt.figure() plt.imshow(cm, interpolation='nearest', cmap=cmap) plt.title(title) plt.colorbar() tick_marks = np.arange(len(classes)) plt.xticks(tick_marks, classes, rotation=45) plt.yticks(tick_marks, classes) fmt = '.2f' if normalize else 'd' thresh = cm.max() / 2. for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): plt.text(j, i, format(cm[i, j], fmt), horizontalalignment="center", color="white" if cm[i, j] > thresh else "black") plt.ylabel('True label') plt.xlabel('Predicted label') plt.tight_layout() # Plot non-normalized confusion matrix class_names = ['0', '1'] plot_confusion_matrix(confmat, classes=class_names, title='Confusion matrix, without normalization') plt.savefig(model_name) #plt.show() #plt.close() #print('f1_score:',F1_score,'ACC_score:',acc,'recall:',recall_score_) #print ('cohen kappa score:', cohen_kappa_score_) print('\n----class report ---:\n', clf_report) metrics.extend([F1_score, acc, recall_score_, cohen_kappa_score_]) metrics.extend(class1_metric) columns = [ 'f1-score', 'acc', 'recall', 'cohen_kappa_score', 'precision-c1', 'recall-c1', 'f-score-c1' ] metrics = dict(zip(columns, metrics)) return metrics
def save_results_to_csv(self, report_lines, all_iter_results): filename = self._results_path header = 'Total_Observations,Train_Size,Graph,Heuristic,K,Start_Date,End_Date,Duration,' \ 'Train_Nodes_in_Graph,Test_Nodes_in_Graph,Number_of_Edges,Number_of_Random' \ 'Guesses,Memory Consumption,Iteration' if not self._results_averaged_on_report: header += ', Precision Class 0, Precision Class 1, Recall Class 0, Recall Class 1, F1 Class 0, ' \ 'F1 Class 1, Support Class 0, Support Class 1 ' else: header += ',Precision,Recall,F1,Support,' header += 'AUC,Avg_Precision,Avg_Recall,Avg_F1,Avg_AUC,Avg_Support,Std_Dev_Precision,' \ 'STD_Dev_Recall,Sev_Dev_F1,Std_Dev_AUC,Std_Dev_Support \n' with open(filename, "w") as text_file: text_file.write(header) for iteration, total_authors, train_size, graph_name, heuristic, k, prediction, actual, start_time, end_time, duration,\ train_nodes_in_graph, test_nodes_in_graph, num_random_guesses, memory_usage, num_edges in report_lines: line = str(total_authors) + ',' + str(train_size) + ',' + str(graph_name) + ',' + heuristic + ',' \ + str(k) + ',' + str(start_time) + ',' + str(end_time) + ',' + str(duration) + ',' \ + str(train_nodes_in_graph) + ',' + str(test_nodes_in_graph) + ',' \ + str(num_edges) + ',' + str(num_random_guesses) + ',' + str(memory_usage)+' ,'+str(iteration) if not self._results_averaged_on_report: results = precision_recall_fscore_support( actual, prediction) prec_a = results[0][0] prec_b = results[0][1] rec_a = results[1][0] rec_b = results[1][1] f1_a = results[2][0] f1_b = results[2][1] support_a = results[3][0] support_b = results[3][1] line += ','+str(prec_a) + ',' + str(prec_b) + ',' + str(rec_a) + ',' + str(rec_b) \ + ',' + str(f1_a) + ', ' + str(f1_b) + ',' + str(support_a) + ',' + str(support_b) else: results = precision_recall_fscore_support(actual, prediction, average='binary') prec = results[0] rec = results[1] f1 = results[2] support = results[3] line += ',' + str(prec) + ',' + str(rec) + ',' + str( f1) + ', ' + str(support) auc = roc_auc_score(actual, prediction) avg_precision = np.mean(all_iter_results[(k, 'precision')]) avg_recall = np.mean(all_iter_results[(k, 'recall')]) avg_f1 = np.mean(all_iter_results[(k, 'f1')]) avg_auc = np.mean(all_iter_results[(k, 'auc')]) if all_iter_results[(k, 'support')] is not None \ and len(all_iter_results[(k, 'support')]) > 0: avg_support = np.mean(all_iter_results[(k, 'support')]) stddev_support = np.std(all_iter_results[(k, 'support')]) else: avg_support = 0.0 stddev_support = 0.0 stddev_precision = np.std(all_iter_results[(k, 'precision')]) stddev_recall = np.std(all_iter_results[(k, 'recall')]) stddev_f1 = np.std(all_iter_results[(k, 'f1')]) stddev_auc = np.std(all_iter_results[(k, 'auc')]) line += ',' + str(auc) + ',' + str(avg_precision) + ',' + str(avg_recall) + ',' + str(avg_f1) + ', '\ + str(avg_auc) + ',' + str(avg_support) + ', '+str(stddev_precision)+', '+ str(stddev_recall)+', '+\ str(stddev_f1)+', '+str(stddev_auc)+','+str(stddev_support) + ' \n' text_file.write(line)
('PreProcess', TextPreprocess()), ('features', StatFeatureVectorizer() ), #used to reduce dimensions pick this one or 'dense' # ('dense', DenseTransformer()), #comment out if svd performed ('clf', LogisticRegression(n_jobs=-1, solver='lbfgs')) ]) #%% Running Logistic Regression cross validation # LR_scores = cross_val_score(LR_pipeline, X_train, y_train, cv=2, scoring="f1_macro") # print(LR_scores) #error analysis print('\n-------------- TFIDF LR -----------------') CVLR = GridSearchCV(LR_pipeline, {}, scoring="f1_macro", cv=2, n_jobs=1) CVLR.fit(X_train, y_train) y_pred = CVLR.predict(X_valid) printed = 0 for (i, label) in enumerate(y_pred): if label != y_valid.iloc[i]: if printed < 20: print(X_valid.iloc[i].encode("utf-8")) print('predicted: %s' % (y_pred[i])) print('labeled: %s' % (y_valid.iloc[i])) printed += 1 results = precision_recall_fscore_support(y_valid, y_pred, average='macro') print('\n\nprecision, recall, f1:') print(results) # precision , recall, fscore #https://www.kaggle.com/evanmiller/pipelines-gridsearch-awesome-ml-pipelines
def run_epoch(self, input_batches, input_set, input_count, topic_params, session, input_labels=None, optimizer=None): loss_sum = 0.0 ppx_sum = 0.0 kld_sum = 0.0 supervised_loss_sum = 0.0 word_count = 0 doc_count = 0 doc_pred = [] doc_labels = [] for idx_batch in input_batches: data_batch, count_batch, mask, label_batch = utils.fetch_data( input_set, input_count, idx_batch, self.vocab_size, topic_params, labels=input_labels) #import pdb; pdb.set_trace() input_feed = { self.x.name: data_batch, self.mask.name: mask, self.label_ids.name: label_batch } if not optimizer is None: _, (loss, kld, supervised_loss, prob) = session.run( (optimizer, [ self.unsupervised_loss, self.kld, self.supervised_loss, self.sup_prob ]), input_feed) else: loss, kld, supervised_loss, prob = session.run([ self.unsupervised_loss, self.kld, self.supervised_loss, self.sup_prob ], input_feed) if topic_params.multilabel: prob_arr = np.asarray(prob) multilabel_pred = np.where(prob_arr >= 0.5, 1, 0) pred = np.ndarray.tolist(multilabel_pred) else: pred = np.argmax(prob, axis=1) assert len(pred) == len(label_batch) == len(mask) for i in range(len(mask)): if mask[i] != 0.0: doc_pred.append(pred[i]) doc_labels.append(label_batch[i]) loss_sum += np.sum(loss) kld_sum += np.sum(kld) / np.sum(mask) supervised_loss_sum += np.sum(supervised_loss) / np.sum(mask) word_count += np.sum(count_batch) # to avoid nan error count_batch = np.add(count_batch, 1e-12) # per document loss ppx_sum += np.sum(np.divide(loss, count_batch)) doc_count += np.sum(mask) assert -1 not in doc_labels if topic_params.multilabel: doc_labels = np.asarray(doc_labels) doc_pred = np.asarray(doc_pred) print_macro_prec, print_macro_recall, print_macro_f1_score, _ = precision_recall_fscore_support( doc_labels, doc_pred, average="macro") #print_micro_prec, print_micro_recall, print_micro_f1_score, _ = precision_recall_fscore_support(doc_labels, doc_pred, average = "micro") print_acc = accuracy_score(doc_labels, doc_pred) print_sup_loss = supervised_loss_sum / len(input_batches) print_ppx = np.exp(loss_sum / word_count) print_ppx_perdoc = np.exp(ppx_sum / doc_count) print_kld = kld_sum / len(input_batches) return print_ppx, print_ppx_perdoc, print_kld, print_sup_loss, print_macro_prec, print_macro_recall, print_macro_f1_score, print_acc
def test(self, train_indices=None, test_indices=None, seen_or_unseen=None, k_fold=None): #Read data if test_indices is not None: self.read_grasp_data_from_indices(train_indices, test_indices) else: self.read_rgbd_data() #Create the variables. self.create_net_var() #Call the network building function. self.cost_function() #Initialize the variables init = tf.global_variables_initializer() saver = tf.train.Saver() tf.get_default_graph().finalize() logs_path = self.logs_path + '_test' start_time = time.time() learn_rate = 0.001 #learn_rate = 0.0001 with tf.Session() as sess: # Test model saver.restore(sess, self.cnn_model_path) pred_score_output = np.array([]).reshape(0, self.classes_num) f2vis_output = np.zeros([ self.testing_samples_num, self.fc1_neurons_num + self.fc2_neurons_num ]) for i in xrange( np.ceil(float(self.testing_samples_num) / self.batch_size).astype(int)): [b_grasp, w_grasp, f2vis_batch_output, pred_batch_output ] = sess.run([ self.biases_grasp['out'], self.weights_grasp['out'], self.feature_to_vis, self.pred ], feed_dict=self.feed_dict_func(False, .0, i)) pred_score_output = np.concatenate( (pred_score_output, pred_batch_output)) start_idx = i * self.batch_size f2vis_output[start_idx:start_idx + len(f2vis_batch_output), :] = f2vis_batch_output print 'pred_score_output: ', pred_score_output print 'self.testing_labels: ', self.testing_labels pred_output = np.copy(pred_score_output) pred_output[pred_output > 0.4] = 1. pred_output[pred_output <= 0.4] = 0. print 'binary pred_output: ', pred_output print 'pred_output.shape: ', pred_output.shape print 'self.testing_labels.shape: ', self.testing_labels.shape pred_errors_all = np.abs(pred_output - self.testing_labels) avg_pred_error_test = np.mean(pred_errors_all) print('avg_pred_error_test:', avg_pred_error_test) #print f2vis_output.shape, self.testing_labels.shape #tsne_vis.tsne_vis(f2vis_output, self.testing_labels[:, 0], '../models/tsne/test_tsne_gt.png', True) #tsne_vis.tsne_vis(f2vis_output, pred_output[:, 0], '../models/tsne/test_tsne_pred.png', True) fig, ax = plt.subplots(figsize=(8, 8)) ax.plot(np.linspace(1, self.testing_samples_num, self.testing_samples_num), self.testing_labels, 'ro', label='gt labels') ax.plot(np.linspace(1, self.testing_samples_num, self.testing_samples_num), pred_output, 'bo', label='pred labels') ax.plot(np.linspace(1, self.testing_samples_num, self.testing_samples_num), pred_errors_all, 'k*', label='pred errors') #plt.legend(handles=[gt_plot, pred_plot, error_plot]) legend = ax.legend(loc='upper right', shadow=True) frame = legend.get_frame() frame.set_facecolor('0.90') fig.savefig('../models/labels_test.png') plt.clf() plt.close(fig) prfc = precision_recall_fscore_support(self.testing_labels, pred_output) print 'precision, recall, fscore, support:' print prfc gt_labels_file_name = '../cross_val/' + seen_or_unseen + '/gt_labels_fold_' + k_fold + '.txt' np.savetxt(gt_labels_file_name, self.testing_labels) pred_score_file_name = '../cross_val/' + seen_or_unseen + '/pred_score_fold_' + k_fold + '.txt' np.savetxt(pred_score_file_name, pred_score_output) roc_fig_name = '../cross_val/' + seen_or_unseen + '/config_net_roc' + k_fold + '.png' plot_roc_pr_curve.plot_roc_curve(self.testing_labels, pred_score_output, roc_fig_name) pr_fig_name = '../cross_val/' + seen_or_unseen + '/config_net_pr' + k_fold + '.png' plot_roc_pr_curve.plot_pr_curve(self.testing_labels, pred_score_output, pr_fig_name) elapsed_time = time.time() - start_time print 'Total testing elapsed_time: ', elapsed_time
def eval_performance(y_true, y_pred, tagnames): pre, rec, f1, support = metrics.precision_recall_fscore_support(y_true, y_pred) print "=== Performance (omitting 'O' class) ===" print "Mean precision: %.02f%%" % (100*sum(pre[1:] * support[1:])/sum(support[1:])) print "Mean recall: %.02f%%" % (100*sum(rec[1:] * support[1:])/sum(support[1:])) print "Mean F1: %.02f%%" % (100*sum(f1[1:] * support[1:])/sum(support[1:]))
def compute_results(self, heuristic, graph_name, iteration, k, predictions, actual, title, duration, results, anchors_percent_size): print(title) if heuristic is not None: print('Link prediciton model: ' + heuristic) print(' K: ' + str(k)) print(' Similarity Function: ' + graph_name) print(' Iteration: ' + str(iteration)) print(' Duration: ' + str(duration)) if len(actual) > 0 and len(predictions) > 0: #report = classification_report(actual, predictions, target_names=[' 0 good actor', '1 bad actor']) report = classification_report(actual, predictions) print(report) print('AUC ' + str(roc_auc_score(actual, predictions))) else: print('No data') auc = roc_auc_score(actual, predictions) results[(k, 'auc')] += [auc] targeted_class_field_name = self._targeted_class_field_name[0] #current_classifier = self._results_dict[targeted_class_field_name][graph_name][k][heuristic] self._result_container.set_result(auc, PerformanceMeasures.AUC, targeted_class_field_name, graph_name, k, heuristic, anchors_percent_size) #original #self._result_container.set_result(auc, PerformanceMeasures.AUC, targeted_class_field_name, graph_name, k, heuristic) #by Lior #current_classifier[PerformanceMeasures.AUC] += auc if not self._results_averaged_on_report: performance = precision_recall_fscore_support(actual, predictions) results[(k, 'precision')] += [ float((performance[0][0] + performance[0][1]) / 2) ] results[(k, 'recall')] += [ float((performance[1][0] + performance[1][1]) / 2) ] results[(k, 'f1')] += [ float((performance[2][0] + performance[2][1]) / 2) ] results[(k, 'support')] += [ float((performance[3][0] + performance[3][1]) / 2) ] else: performance = precision_recall_fscore_support(actual, predictions, average='binary') precision = performance[0] results[(k, 'precision')] += [precision] self._result_container.set_result(precision, PerformanceMeasures.PRECISION, targeted_class_field_name, graph_name, k, heuristic, anchors_percent_size) #original #self._result_container.set_result(precision, PerformanceMeasures.PRECISION, targeted_class_field_name, # graph_name, k, # heuristic) #By Lior recall = performance[1] results[(k, 'recall')] += [recall] self._result_container.set_result(recall, PerformanceMeasures.RECALL, targeted_class_field_name, graph_name, k, heuristic, anchors_percent_size) #original #self._result_container.set_result(recall, PerformanceMeasures.RECALL, targeted_class_field_name, # graph_name, k, # heuristic) #by Lior f1 = performance[2] results[(k, 'f1')] += [f1] results[(k, 'support')] = None accuracy = accuracy_score(actual, predictions) self._result_container.set_result(accuracy, PerformanceMeasures.ACCURACY, targeted_class_field_name, graph_name, k, heuristic, anchors_percent_size) # self._result_container.set_result(accuracy, PerformanceMeasures.ACCURACY, targeted_class_field_name, # graph_name, k, # heuristic) confusion_matrix_score = confusion_matrix(actual, predictions) self._result_container.set_result( confusion_matrix_score, PerformanceMeasures.CONFUSION_MATRIX, targeted_class_field_name, graph_name, k, heuristic, anchors_percent_size) # self._result_container.set_result(confusion_matrix_score, PerformanceMeasures.CONFUSION_MATRIX, # targeted_class_field_name, # graph_name, k, # heuristic) num_of_correct_instances, num_of_incorrect_instances = self._result_container.calculate_correctly_and_not_correctly_instances( confusion_matrix_score) self._result_container.set_result( num_of_correct_instances, PerformanceMeasures.CORRECTLY_CLASSIFIED, targeted_class_field_name, graph_name, k, heuristic, anchors_percent_size) # self._result_container.set_result(num_of_correct_instances, PerformanceMeasures.CORRECTLY_CLASSIFIED, # targeted_class_field_name, # graph_name, k, # heuristic) self._result_container.set_result( num_of_incorrect_instances, PerformanceMeasures.INCORRECTLY_CLASSIFIED, targeted_class_field_name, graph_name, k, heuristic, anchors_percent_size) # self._result_container.set_result(num_of_incorrect_instances, PerformanceMeasures.INCORRECTLY_CLASSIFIED, # targeted_class_field_name, # graph_name, k, # heuristic) self._result_container.set_result( num_of_incorrect_instances, PerformanceMeasures.SELECTED_FEATURES, targeted_class_field_name, graph_name, k, heuristic, anchors_percent_size)
X_Test = sequence.pad_sequences(tokenisedTest, maxlen=max_review_length,padding='post') #Radical radicalModel = Sequential() radicalModel.add(Embedding(vocabSize, 100, input_length=max_review_length,weights=[embedding_matrix], trainable=False)) radicalModel.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2)) radicalModel.add(Dense(1, activation='sigmoid')) radicalModel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) radicalModel.fit(X_Train,radicalTrain,epochs=10, batch_size=100) radicalScore = radicalModel.evaluate(X_Test,radicalTest,verbose = 100) accuRadicalLstm = radicalScore[1] print("\nRadical Training Done for Iteration",iteration) positiveRadical = [x for x in radicalTest if x == 1] predictRadical = radicalModel.predict_classes(X_Test, verbose = 1) positivePredRadical = [x for x in predictRadical if x > 0] prec, recall, fscore, support = precision_recall_fscore_support(radicalTest, predictRadical) print("Number of positive Examples : ",len(positiveRadical), "\nratio : ", (len(positiveRadical) / len(radicalTest)), "\nPositive Predicted : ", len(positivePredRadical), "\naccuracy : ", accuRadicalLstm, "\nwrongness : ", 1 - accuRadicalLstm,"\n\nPrecision : ",prec,"\nRecall : ", recall, "\nf1Score : ", fscore, "\nsupport : ", support ) gRadicalAccu += accuRadicalLstm gPositivePredRadical += len(positivePredRadical) gPrecision[0] += prec[0] gPrecision[1] += prec[1] gRecall[0] += recall[0] gRecall[1] += recall[1] gFScore[0] += fscore[0] gFScore[1] += fscore[1] with timer("final Output"):
def classification_report(classificator_type, y_true, y_pred): ''' Computes clasification metrics :param y_true - original class label :param y_pred - predicted class label :return presicion, recall for each class; micro_f1 measure, macro_f1 measure ''' report = classificator_type + '\n' last_line_heading = 'avg / total' final_line_heading = 'final score' labels = unique_labels(y_true, y_pred) width = len(last_line_heading) target_names = ['{0}'.format(l) for l in labels] headers = ["precision", "recall", "f1-score", "support"] fmt = '%% %ds' % width # first column: class name fmt += ' ' fmt += ' '.join(['% 9s' for _ in headers]) fmt += '\n' headers = [""] + headers report += fmt % tuple(headers) report += '\n' p, r, f1, s = precision_recall_fscore_support(y_true, y_pred, labels=labels, average=None) f1_macro = 0 precision_macro = 0 recall_macro = 0 for i, label in enumerate(labels): values = [target_names[i]] f1_macro += f1[i] precision_macro += p[i] recall_macro += r[i] for v in (p[i], r[i], f1[i]): values += ["{0:0.5f}".format(v)] values += ["{0}".format(s[i])] report += fmt % tuple(values) report += '\n' # compute averages values = [last_line_heading] for v in (np.average(p, weights=s), np.average(r, weights=s), np.average(f1, weights=s)): values += ["{0:0.5f}".format(v)] values += ['{0}'.format(np.sum(s))] report += fmt % tuple(values) values = [final_line_heading] for v in (precision_macro, recall_macro, f1_macro): values += ["{0:0.5f}".format(v / labels.size)] values += ['{0}'.format(np.sum(s))] report += fmt % tuple(values) return report
def run_graph(self): # INPUTs is_training = tf.placeholder(tf.bool, shape=(), name='bool_train') x_char_ngram_indices = tf.placeholder(tf.int32, shape=[None, self.params['max_domain_segments_len'], self.max_num_charngrams], name='embedding') x_suffix = tf.placeholder(tf.float32, shape=[None, self.params['num_suffix']], name='suffix') seq_len = tf.placeholder(tf.int32, shape=[None], name='length') sample_weights = tf.placeholder(tf.float32, shape=[None], name='weight') y = tf.placeholder(tf.int32, shape=[None], name='target') # Each entry in y must be an index in [0, num_classes) # embedding layers # Look up embeddings for inputs. embeddings = tf.Variable(tf.random_uniform([len(self.charngram2index), embed_dimen], -1.0, 1.0)) embed = tf.nn.embedding_lookup(embeddings, x_char_ngram_indices) mask = tf.placeholder(tf.float32, shape=[None, self.params['max_domain_segments_len'], self.max_num_charngrams], name='embed_mask') mask = tf.expand_dims(mask, axis=-1) mask = tf.tile(mask, [1,1,1,embed_dimen]) x_embed = tf.multiply(embed, mask) # x_embed.shape: (None, self.params['max_domain_segments_len'], # self.max_num_charngrams, embed_dimen) x_embed = tf.reduce_mean(x_embed, 2) domain_vectors = [] if 'RNN' in type: rnn_cell = tf.contrib.rnn.BasicRNNCell(n_rnn_neurons, activation=tf.nn.tanh) # The shape of last_states should be [batch_size, n_lstm_neurons] _, domain_vec_rnn = tf.nn.dynamic_rnn(rnn_cell, x_embed, sequence_length=seq_len, dtype=tf.float32, time_major=False) domain_vec_rnn = tf.layers.dropout(domain_vec_rnn, dropout_rate, training=is_training) domain_vectors.append(domain_vec_rnn) if 'CNN' in type: pooled_outputs = [] for filter_size in filter_sizes: # Define and initialize filters filter_shape = [filter_size, embed_dimen, 1, num_filters] W_filter = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1)) # initialize the filters' weights b_filter = tf.Variable(tf.constant(0.1, shape=[num_filters])) # initialize the filters' biases # The conv2d operation expects a 4-D tensor with dimensions corresponding to batch, width, height and channel. # The result of our embedding doesn’t contain the channel dimension # So we add it manually, leaving us with a layer of shape [None, sequence_length, embedding_size, 1]. x_embed_expanded = tf.expand_dims(x_embed, -1) conv = tf.nn.conv2d(x_embed_expanded, W_filter, strides=[1, 1, 1, 1], padding="VALID") # Apply nonlinearity h = tf.nn.relu(tf.nn.bias_add(conv, b_filter), name="relu") pooled = tf.nn.max_pool(h, ksize=[1, self.params['max_domain_segments_len'] - filter_size + 1, 1, 1], strides=[1, 1, 1, 1], padding='VALID') pooled_outputs.append(pooled) # Combine all the pooled features h_pool = tf.concat(pooled_outputs, axis=3) num_filters_total = num_filters * len(filter_sizes) domain_vec_cnn = tf.reshape(h_pool, [-1, num_filters_total]) domain_vec_cnn = tf.layers.dropout(domain_vec_cnn, dropout_rate, training=is_training) domain_vectors.append(domain_vec_cnn) # concatenate suffix one-hot and the abstract representation of the domains segments # The shape of cat_layer should be [batch_size, n_lstm_neurons+self.params['num_suffix']] cat_layer = tf.concat(domain_vectors + [x_suffix], -1) # print(cat_layer.get_shape()) logits = cat_layer for _ in range(n_fc_layers): logits = tf.contrib.layers.fully_connected(logits, num_outputs=n_rnn_neurons, activation_fn=act_fn) logits = tf.layers.dropout(logits, dropout_rate, training=is_training) logits = tf.contrib.layers.fully_connected(logits, self.params['num_targets'], activation_fn=act_fn) if class_weighted: crossentropy = tf.losses.sparse_softmax_cross_entropy(labels=y, logits=logits, weights=sample_weights) else: crossentropy = tf.losses.sparse_softmax_cross_entropy(labels=y, logits=logits) loss_mean = tf.reduce_mean(crossentropy) optimizer = tf.train.AdamOptimizer(learning_rate=lr_rate) training_op = optimizer.minimize(loss_mean) prediction = tf.argmax(logits, axis=-1) is_correct = tf.nn.in_top_k(logits, y, 1) # logits are unscaled, but here we only care the argmax n_correct = tf.reduce_sum(tf.cast(is_correct, tf.float32)) accuracy = tf.reduce_mean(tf.cast(is_correct, tf.float32)) init = tf.global_variables_initializer() ''' For TensorBoard ''' ''' now = datetime.now().strftime("%Y%m%d%H%M%S") root_log_dir = 'tf_logs' logdir = OUTPUT_DIR + root_log_dir + "/run-" + now + "/" loss_summary = tf.summary.scalar('loss_mean', loss_mean) acc_summary = tf.summary.scalar('accuracy', accuracy) file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph()) ''' with tf.Session() as sess: init.run() n_total_batches = int(np.ceil(len(self.domains_train) / batch_size)) test_fscore_history = [] for epoch in range(1, n_epochs + 1): # model training n_batch = 0 for X_batch_embed, X_batch_mask, domain_actual_lens, X_batch_suf, sample_weights, y_batch in self.next_batch(self.domains_train): _, acc_batch_train, loss_batch_train, prediction_train = sess.run([training_op, accuracy, loss_mean, prediction], feed_dict={ 'bool_train:0': True, 'embedding:0': X_batch_embed, 'embed_mask:0': X_batch_mask, 'suffix:0': X_batch_suf, 'length:0': domain_actual_lens, 'weight:0': sample_weights, 'target:0': y_batch}) n_batch += 1 if epoch < 2: # print(prediction_train) print("Epoch %d - Batch %d/%d: loss = %.4f, accuracy = %.4f" % (epoch, n_batch, n_total_batches, loss_batch_train, acc_batch_train)) # evaluation on training data eval_nodes = [n_correct, loss_mean, is_correct, prediction] print() print("========== Evaluation at Epoch %d ==========" % epoch) loss_train, acc_train, _, _ = self.evaluate(self.domains_train, sess, eval_nodes) print("*** On Training Set:\tloss = %.6f\taccuracy = %.4f" % (loss_train, acc_train)) # evaluation on validation data loss_val, acc_val, _, _ = self.evaluate(self.domains_val, sess, eval_nodes) print("*** On Validation Set:\tloss = %.6f\taccuracy = %.4f" % (loss_val, acc_val)) # evaluate on test data loss_test, acc_test, is_correct_test, pred_test = self.evaluate(self.domains_test, sess, eval_nodes) print("*** On Test Set:\tloss = %.6f\taccuracy = %.4f" % (loss_test, acc_test)) print() print("Macro average:") precisions_macro, recalls_macro, fscores_macro, _ = precision_recall_fscore_support( [category2index[domain['categories'][1]] for domain in self.domains_test], pred_test, average='macro') print("Precision (macro): %.4f, Recall (macro): %.4f, F-score (macro): %.4f" % (precisions_macro, recalls_macro, fscores_macro)) print() if not test_fscore_history or fscores_macro > max(test_fscore_history): # the accuracy of this epoch is the largest print("Classification Performance on individual classes:") precisions_none, recalls_none, fscores_none, supports_none = precision_recall_fscore_support( [category2index[domain['categories'][1]] for domain in self.domains_test], pred_test, average=None) print(tabulate(zip((categories[i] for i in range(len(precisions_none))), precisions_none, recalls_none, fscores_none, supports_none), headers=['category', 'precision', 'recall', 'f-score', 'support'], tablefmt='orgtbl')) # output all incorrect_prediction with open(os.path.join(OUTPUT_DIR, 'incorrect_predictions.csv'), 'w') as outfile: csv_writer = csv.writer(outfile) csv_writer.writerow(('RAW_DOMAIN', 'SEGMENTED_DOMAIN', 'TRUE_CATEGORY', 'PRED_CATEGORY')) for correct, pred_catIdx, domain in zip(is_correct_test, pred_test, self.domains_test): if correct: continue csv_writer.writerow((domain['raw_domain'], domain['segmented_domain'], domain['categories'][1], categories[pred_catIdx])) test_fscore_history.append(fscores_macro)
def evaluate(self, data, y_map, tracker_fn='tracker.csv', channel_probs=False, predict_english_only=False): print(channel_probs) best_epochs = int(re.findall('_e(\d+)_', self.best_fn)[0]) try: model = load_model(os.path.join(self.modeldir, self.best_fn)) except: model = load_model(os.path.join(self.modeldir, self.best_fn), custom_objects={'f1': ut.f1}) y_decoder = {v: k for k, v in y_map.items()} X_test = data['test']['xarr'] y_test = data['test']['yarr'] y_test_labels = [np.argmax(cat) for cat in y_test] #predict #y_pred = model.predict(data.X_test_seq,batch_size=self.batch_size) if channel_probs == True: all_lang_preds = [] for arr in X_test: y_pred = model.predict(arr, batch_size=self.batch_size) #y_pred_labels=[np.argmax(pred) for pred in y_pred] all_lang_preds.append(y_pred) #all_lang_preds.append(y_pred_labels) y_pred_labels = [] for i, probs in enumerate(zip(*all_lang_preds)): y_pred_labels.append( np.argmax(np.average(np.array(probs), axis=0))) elif predict_english_only == True: y_pred = model.predict(X_test[0], batch_size=self.batch_size) y_pred_labels = [np.argmax(pred) for pred in y_pred] else: y_pred = model.predict(X_test, batch_size=self.batch_size) y_pred_labels = [np.argmax(pred) for pred in y_pred] #evaluate conf_matrix = confusion_matrix(y_test_labels, y_pred_labels) acc = accuracy_score(y_test_labels, y_pred_labels) w_prec, w_rec, w_f1, _ = precision_recall_fscore_support( y_test_labels, y_pred_labels, average='weighted') mic_prec, mic_rec, mic_f1, _ = precision_recall_fscore_support( y_test_labels, y_pred_labels, average='micro') mac_prec, mac_rec, mac_f1, _ = precision_recall_fscore_support( y_test_labels, y_pred_labels, average='macro') results = { 'model_filepath': self.best_fn, 'languages': self.x_cols, 'model_name': self.model_name, 'date': '-', 'time': '-', 'train_time': self.train_time, 'best_epochs': best_epochs, 'accuracy': acc, 'batch_size': self.batch_size, 'total_epochs': self.total_epochs, 'summary': self.summary, 'precision (weighted)': w_prec, 'f1 (weighted)': w_f1, 'precision (micro)': mic_prec, 'recall (micro)': mic_rec, 'f1 (micro)': mic_f1, 'precision (macro)': mac_prec, 'recall (macro)': mac_rec, 'f1 (macro)': mac_f1, 'confusion_matrix': conf_matrix } tracker = pd.read_csv(os.path.join(self.modeldir, tracker_fn)) tracker = tracker.append(results, ignore_index=True) tracker.to_csv(os.path.join(self.modeldir, tracker_fn), index=False)
def train_basic(dirpath_vector, dirpath_output, verbose=True): logger = utils.get_logger() x_train = np.genfromtxt(dirpath_vector + '/phylum/train.csv', delimiter='\n', dtype=None, encoding=None) x_test = np.genfromtxt(dirpath_vector + '/phylum/test.csv', delimiter='\n', dtype=None, encoding=None) x_val = np.genfromtxt(dirpath_vector + '/phylum/val.csv', delimiter='\n', dtype=None, encoding=None) arr = [] arr1 = [] arr2 = [] for item in x_train[1:]: arr.append(ordinal_encoder(string_to_array(item.split(",")[3]))) for item in x_test[1:]: arr1.append(ordinal_encoder(string_to_array(item.split(",")[3]))) for item in x_val[1:]: arr2.append(ordinal_encoder(string_to_array(item.split(",")[3]))) maxi = 0 for item in arr: if len(item) > maxi: maxi = len(item) final1 = np.zeros((x_train.shape[0] - 1, maxi)) count = 0 for item in arr: final1[count][:len(item)] = item count += 1 maxi1 = 0 for item in arr1: if len(item) > maxi1: maxi1 = len(item) final2 = np.zeros((x_test.shape[0] - 1, maxi1)) count = 0 for item in arr1: final2[count][:len(item)] = item count += 1 maxi2 = 0 for item in arr2: if len(item) > maxi2: maxi2 = len(item) final3 = np.zeros((x_val.shape[0] - 1, maxi2)) count = 0 for item in arr2: final3[count][:len(item)] = item count += 1 hf = h5py.File(dirpath_vector + '/phylum/ordinal.h5', 'w') hf.create_dataset('dataset_1', data=final1) hf.create_dataset('dataset_2', data=final2) hf.create_dataset('dataset_3', data=final3) hf.close() x_train = np.genfromtxt(dirpath_vector + '/class/train.csv', delimiter='\n', dtype=None, encoding=None) x_test = np.genfromtxt(dirpath_vector + '/class/test.csv', delimiter='\n', dtype=None, encoding=None) x_val = np.genfromtxt(dirpath_vector + '/class/val.csv', delimiter='\n', dtype=None, encoding=None) arr = [] arr1 = [] arr2 = [] for item in x_train[1:]: arr.append(ordinal_encoder(string_to_array(item.split(",")[3]))) for item in x_test[1:]: arr1.append(ordinal_encoder(string_to_array(item.split(",")[3]))) for item in x_val[1:]: arr2.append(ordinal_encoder(string_to_array(item.split(",")[3]))) maxi = 0 for item in arr: if len(item) > maxi: maxi = len(item) final1 = np.zeros((x_train.shape[0] - 1, maxi)) count = 0 for item in arr: final1[count][:len(item)] = item count += 1 maxi1 = 0 for item in arr1: if len(item) > maxi1: maxi1 = len(item) final2 = np.zeros((x_test.shape[0] - 1, maxi1)) count = 0 for item in arr1: final2[count][:len(item)] = item count += 1 maxi2 = 0 for item in arr2: if len(item) > maxi2: maxi2 = len(item) final3 = np.zeros((x_val.shape[0] - 1, maxi2)) count = 0 for item in arr2: final3[count][:len(item)] = item count += 1 hf = h5py.File(dirpath_vector + '/class/ordinal.h5', 'w') hf.create_dataset('dataset_1', data=final1) hf.create_dataset('dataset_2', data=final2) hf.create_dataset('dataset_3', data=final3) hf.close() hf = h5py.File(dirpath_vector + '/phylum/ordinal.h5', 'r') n1 = hf.get('dataset_1') n2 = hf.get('dataset_2') n3 = hf.get('dataset_3') X = np.array(n1) Y = np.array(n2) V = np.array(n3) hf.close() lab = np.genfromtxt(dirpath_vector + '/phylum/train.csv', delimiter='\n', dtype=None, encoding=None) lab1 = np.genfromtxt(dirpath_vector + '/phylum/test.csv', delimiter='\n', dtype=None, encoding=None) lab2 = np.genfromtxt(dirpath_vector + '/phylum/val.csv', delimiter='\n', dtype=None, encoding=None) labels = [] i = 0 for item in lab[1:]: if item.split(",")[0][0] == "A": labels.append(0) elif item.split(",")[0][0] == "F": labels.append(1) else: labels.append(2) i += 1 labels1 = [] i = 0 for item in lab1[1:]: if item.split(",")[0][0] == "A": labels1.append(0) elif item.split(",")[0][0] == "F": labels1.append(1) else: labels1.append(2) i += 1 labels2 = [] i = 0 for item in lab2[1:]: if item.split(",")[0][0] == "A": labels2.append(0) elif item.split(",")[0][0] == "F": labels2.append(1) else: labels2.append(2) i += 1 label = np.array(labels) label1 = np.array(labels1) label2 = np.array(labels2) clf2 = SVC(kernel='rbf') clf = RandomForestClassifier() clf2.fit(X, label) clf.fit(X, label) preds2 = clf2.predict(X) preds = clf.predict(X) scores = clf2.decision_function(X) scores2 = clf.predict(X) score = np.amax(scores, axis=1) fpr, tpr, thresholds = roc_curve(label, score, pos_label=2) fpr2, tpr2, thresholds2 = roc_curve(label, scores2, pos_label=2) match2 = 0 for i in range(preds2.shape[0]): if preds2[i] == label[i]: match2 += 1 accuracy2 = float(match2) / preds2.shape[0] p, r, f1, s = precision_recall_fscore_support(label, preds2, average='weighted') match = 0 for i in range(preds.shape[0]): if preds[i] == label[i]: match += 1 accuracy = float(match) / preds.shape[0] p2, r2, f12, s = precision_recall_fscore_support(label, preds, average='weighted') C = confusion_matrix(label, preds2) logger.info( 'Train Accuracy, precision, recall and F1 Score for SVM model for phylum level is {:.3f}, {:.3f}, {:.3f}, {:.3f}' .format(accuracy2, p, r, f1)) logger.info( 'Train Accuracy, precision, recall and F1 Score for Random Forest model for phylum level is {:.3f}, {:.3f}, {:.3f}, {:.3f}' .format(accuracy, p2, r2, f12)) hf = h5py.File(dirpath_vector + '/class/ordinal.h5', 'r') n1 = hf.get('dataset_1') n2 = hf.get('dataset_2') n3 = hf.get('dataset_3') X = np.array(n1) Y = np.array(n2) V = np.array(n3) hf.close() lab = np.genfromtxt(dirpath_vector + '/class/train.csv', delimiter='\n', dtype=None, encoding=None) lab1 = np.genfromtxt(dirpath_vector + '/class/test.csv', delimiter='\n', dtype=None, encoding=None) lab2 = np.genfromtxt(dirpath_vector + '/class/val.csv', delimiter='\n', dtype=None, encoding=None) labels = [] i = 0 for item in lab[1:]: labels.append(int(item.split(",")[2])) i += 1 labels1 = [] i = 0 for item in lab1[1:]: labels1.append(int(item.split(",")[2])) i += 1 labels2 = [] i = 0 for item in lab2[1:]: labels2.append(int(item.split(",")[2])) i += 1 label = np.array(labels) label1 = np.array(labels1) label2 = np.array(labels2) clf2 = RandomForestClassifier() clf = SVC(kernel='rbf') clf2.fit(X, label) clf.fit(X, label) preds2 = clf2.predict(X) preds = clf.predict(X) scores = clf2.predict(X) scores1 = clf.decision_function(X) score = np.amax(scores1, axis=1) fpr, tpr, thresholds = roc_curve(label, scores, pos_label=2) fpr2, tpr2, thresholds2 = roc_curve(label, score, pos_label=2) match2 = 0 for i in range(preds2.shape[0]): if preds2[i] == label[i]: match2 += 1 accuracy2 = float(match2) / preds2.shape[0] p, r, f1, s = precision_recall_fscore_support(label, preds2, average='weighted') C = confusion_matrix(label, preds2) match = 0 for i in range(preds.shape[0]): if preds[i] == label[i]: match += 1 accuracy = float(match) / preds.shape[0] p2, r2, f12, s = precision_recall_fscore_support(label, preds, average='weighted') logger.info( 'Train Accuracy, precision, recall and F1 Score for SVM model for phylum level is {:.3f}, {:.3f}, {:.3f}, {:.3f}' .format(accuracy, p2, r2, f12)) logger.info( 'Train Accuracy, precision, recall and F1 Score for Random Forest model for phylum level is {:.3f}, {:.3f}, {:.3f}, {:.3f}' .format(accuracy2, p, r, f1))