def test_prf(fn1,fn2,sth,L): y_true=[] y_score=[] edges_1=prep.read_edges(fn1) edges_2=prep.read_edges(fn2) predict_set={} for key in sth.keys(): predict_set[key]=predict_set.get(key,0.)+sth[key] predict_set=sorted(predict_set.iteritems(),key=lambda d:d[1],reverse=True)# threshold=predict_set[L][1] for i in edges_1: if sth[i]>threshold: y_score.append(1) else: y_score.append(0) for i in edges_1: if i not in edges_2: y_true.append(0) else: y_true.append(1) print classification_report(y_true,y_score) print auc_score(y_true,y_score)
def eval_model(): comments, labels = load_extended_data() clf1 = build_base_model() clf2 = build_elasticnet_model() clf3 = build_stacked_model() clf4 = build_nltk_model() models = [clf1, clf2, clf3, clf4] #models = [clf1] cv = ShuffleSplit(len(comments), n_iterations=5, test_size=0.2, indices=True) scores = [] for train, test in cv: probs_common = np.zeros((len(test), 2)) for clf in models: X_train, y_train = comments[train], labels[train] X_test, y_test = comments[test], labels[test] clf.fit(X_train, y_train) probs = clf.predict_proba(X_test) print("score: %f" % auc_score(y_test, probs[:, 1])) probs_common += probs probs_common /= 4. scores.append(auc_score(y_test, probs_common[:, 1])) print("combined score: %f" % scores[-1]) print(np.mean(scores), np.std(scores))
def test_thresholded_scorers(): """Test scorers that take thresholds.""" X, y = make_blobs(random_state=0, centers=2) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) clf = LogisticRegression(random_state=0) clf.fit(X_train, y_train) score1 = SCORERS['roc_auc'](clf, X_test, y_test) score2 = auc_score(y_test, clf.decision_function(X_test)) score3 = auc_score(y_test, clf.predict_proba(X_test)[:, 1]) assert_almost_equal(score1, score2) assert_almost_equal(score1, score3) logscore = SCORERS['log_loss'](clf, X_test, y_test) logloss = log_loss(y_test, clf.predict_proba(X_test)) assert_almost_equal(-logscore, logloss) # same for an estimator without decision_function clf = DecisionTreeClassifier() clf.fit(X_train, y_train) score1 = SCORERS['roc_auc'](clf, X_test, y_test) score2 = auc_score(y_test, clf.predict_proba(X_test)[:, 1]) assert_almost_equal(score1, score2) # Test that an exception is raised on more than two classes X, y = make_blobs(random_state=0, centers=3) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) clf.fit(X_train, y_train) assert_raises(ValueError, SCORERS['roc_auc'], clf, X_test, y_test)
def test_score_scale_invariance(): # Test that average_precision_score and auc_score are invariant by # the scaling or shifting of probabilities y_true, _, probas_pred = make_prediction(binary=True) roc_auc = auc_score(y_true, probas_pred) roc_auc_scaled = auc_score(y_true, 100 * probas_pred) roc_auc_shifted = auc_score(y_true, probas_pred - 10) assert_equal(roc_auc, roc_auc_scaled) assert_equal(roc_auc, roc_auc_shifted) pr_auc = average_precision_score(y_true, probas_pred) pr_auc_scaled = average_precision_score(y_true, 100 * probas_pred) pr_auc_shifted = average_precision_score(y_true, probas_pred - 10) assert_equal(pr_auc, pr_auc_scaled) assert_equal(pr_auc, pr_auc_shifted)
def ScoreClassifier(features, labels, clf=None, score_func=None): """Test a learned classifier. :type callable score_func: Scoring function (one of accuracy_scorer or auc_scorer). This is not a score function from sklearn.metrics. :rtype: float :returns: Accuracy of classifier on test data. """ # Note: type(clf) will be 'instance' if clf is a learned classifier. If # instead type(clf) is 'type', then it is assumed to be the class of learning # algorithm to apply. if clf is None or type(clf) == type: mask = ChooseTrainingSet(labels, 0.5) clf = FitClassifier(features[mask], labels[mask], algorithm=clf) features = features[~mask] labels = labels[~mask] features = features.astype(float) score_func = score_func or 'accuracy' if isinstance(score_func, basestring): score_func = score_func.lower() if score_func == 'accuracy': score_func = accuracy_score elif score_func == 'auc': predictions = clf.decision_function(features) return auc_score(labels, predictions), predictions elif not callable(score_func): raise ValueError("Score function must be a string or a callable.") predictions = clf.predict(features) return score_func(labels, predictions), predictions
def auc(test_data, index, reverse, test_file): pred = [x[index] for x in test_data] if reverse: pred = [ x * -1 for x in pred] testing_Y = [x[0] for x in test_data] print "AUC: \n%f\n" % metrics.auc_score(testing_Y, pred)
def run_cv(x,y,reg,cv): ''' returns mean AUC for this reg using cv splits.''' scores = [] for sp in cv: reg.fit(x[sp[0],:],y[sp[0]]) scores.append(auc_score(y[sp[1]],reg.predict_proba(x[sp[1],:])[:,1])) return np.mean(scores)
def classification_metrics (targets, preds, probs=None): # if probs != None and len(probs) > 0: # fpr, tpr, thresholds = roc_curve(targets, probs[:, 1], 1) # roc_auc = auc_score(fpr, tpr) # else: # fpr, tpr, thresholds = roc_curve(targets, preds, 1) # roc_auc = auc_score(targets, preds) auc = 0 if len(targets) > 1: auc = auc_score(targets, preds) cm = confusion_matrix(targets, preds) #accuracy acc = accuracy_score(targets, preds) #recall? True Positive Rate or Sensitivity or Recall sens = recall_score(targets, preds) #precision prec = precision_score(targets, preds) #f1-score f1 = f1_score(targets, preds, np.unique(targets), 1) tnr = 0.0 spec = 0.0 #True Negative Rate or Specificity (tn / (tn+fp)) if len(cm) == 2: if (cm[0,0] + cm[0,1]) != 0: spec = float(cm[0,0])/(cm[0,0] + cm[0,1]) return acc, sens, spec, prec, f1, auc
def test_auc(): probs = numpy.ravel([test_pred_proba_i(i)[:,1] for i in xrange(n_test_batches)]) if numpy.all(test_set_y.get_value()) and numpy.all(probs): return 1. if numpy.all(test_set_y.get_value() == 0) and numpy.all(probs == 0): return 0. return auc_score(test_set_y.get_value()[:n_test_batches*batch_size], probs)
def evalSymbReg(individual): # Transform the tree expression in a callable function func = toolbox.lambdify(expr=individual) # Evaluate the sum of squared difference between the expression # and the real function : x**4 + x**3 + x**2 + x #X=[[1.0000,0.9231,1.0000,1.0000,1.0000,1.0000,0.9091,1.0000]] #X.append([1.0000,0.9231,1.0000,1.0000,0.8333,0.5000,0.9091,0.8333]) #X.append([1.0000,0.9231,1.0000,1.0000,0.8333,1.0000,0.9091,0.8333]) #X.append([0.0000,0.9231,0.0000,0.0000,0.8333,0.5000,0.9091,0.8333]) #X=[[1.0000,0,1.0000,1.0000,1.0000,1.0000,0,1.0000]] #X.append([1.0000,0,1.0000,1.0000,0,0.5000,0,0]) #X.append([1.0000,0,1.0000,1.0000,0,1.0000,0,0]) #X.append([1.0000,1,1.0000,1.0000,0,0.5000,0,1]) #X.append([0.0000,1,1.0000,1.0000,0,0.5000,0,1]) #L #A=[1,1,1,0,1]; t=0 a=0 global co co+=1 # print co if(co>6000): global A,X A,X=getdata() co=0 preds=[] for x in X: preds.append(func(x[0],x[1],x[2],x[3],x[4],x[5],x[6],x[7])) #t+=1 auc = 1-metrics.auc_score(A, preds) return auc,
def bagging(): from sklearn.feature_selection import SelectPercentile, chi2 comments, dates, labels = load_data() select = SelectPercentile(score_func=chi2, percentile=4) clf = LogisticRegression(tol=1e-8, penalty='l2', C=7) #clf = BaggingClassifier(logr, n_estimators=50) countvect_char = TfidfVectorizer(ngram_range=(1, 5), analyzer="char", binary=False) countvect_word = TfidfVectorizer(ngram_range=(1, 3), analyzer="word", binary=False) badwords = BadWordCounter() ft = FeatureStacker([("badwords", badwords), ("chars", countvect_char), ("words", countvect_word)]) #ft = TextFeatureTransformer() pipeline = Pipeline([('vect', ft), ('select', select), ('logr', clf)]) cv = ShuffleSplit(len(comments), n_iterations=20, test_size=0.2, indices=True) scores = [] for train, test in cv: X_train, y_train = comments[train], labels[train] X_test, y_test = comments[test], labels[test] pipeline.fit(X_train, y_train) probs = pipeline.predict_proba(X_test) scores.append(auc_score(y_test, probs[:, 1])) print("score: %f" % scores[-1]) print(np.mean(scores), np.std(scores))
def model_generate_level1(bestc, features, X_train, y) : ntrain = X_train.shape[0] newdata = np.zeros(ntrain) X_train, keymap = utility.OneHotEncoder(X_train[:,features]) model = linear_model.LogisticRegression() model.C = bestc cvscores = [] cvgen = cross_validation.KFold(ntrain, 10, random_state=utility.SEED) for train_inds, test_inds in cvgen : X_cvtrain = X_train[train_inds] X_cvtest = X_train[test_inds] y_cvtrain = y[train_inds] y_cvtest = y[test_inds] model.fit(X_cvtrain, y_cvtrain) pred_cvtest = model.predict_proba(X_cvtest)[:,1] cvscore = metrics.auc_score(y_cvtest, pred_cvtest) cvscores.append(cvscore) newdata[test_inds] = pred_cvtest print "Average CV Score: {}".format(np.mean(cvscores)) return newdata
def run_fest_test(festpath="/Users/bjcohen/dev/fest", **kwargs): """ -c <int> : committee type: 1 bagging 2 boosting (default) 3 random forest -d <int> : maximum depth of the trees (default: 1000) -e : report out of bag estimates (default: no) -n <float>: relative weight for the negative class (default: 1) -p <float>: parameter for random forests: (default: 1) (ratio of features considered over sqrt(features)) -t <int> : number of trees (default: 100) """ idstr = "".join(map(lambda (f, v): f + str(v), kwargs.items())) ret = call( [ os.path.join(festpath, "festlearn"), " ".join(map(lambda (f, v): "-" + f + str(v), kwargs.items())), os.path.join("..", "data", "train_3way_-27000.libsvm"), os.path.join("..", "data", "fest_%s_-27000.model" % idstr), ] ) if ret != 0: raise Exception() ret = call( [ os.path.join(festpath, "festclassify"), os.path.join("..", "data", "train_3way_-27000.libsvm"), os.path.join("..", "data", "fest_%s_-27000.model" % idstr), os.path.join("..", "data", "pred_fest_train_-27000_%s" % idstr), ] ) if ret != 0: raise Exception() ret = call( [ os.path.join(festpath, "festclassify"), os.path.join("..", "data", "train_3way_27000-.libsvm"), os.path.join("..", "data", "fest_%s_-27000.model" % idstr), os.path.join("..", "data", "pred_fest_train_27000-_%s" % idstr), ] ) if ret != 0: raise Exception() tr_score = auc_score(ACTION[:27000], pd.read_table("../data/pred_fest_train_-27000_%s" % idstr, header=None)) te_score = auc_score(ACTION[27000:], pd.read_table("../data/pred_fest_train_27000-_%s" % idstr, header=None)) return (tr_score, te_score)
def calculatePrediction(): dTr = loadFile('../data/train.csv') y_train = dTr[0] X_train_A = dTr[1] X_train_B = dTr[2] dTes = loadFileTest('../data/test.csv') X_test_A = dTes[0] X_test_B = dTes[1] print "train size: {0} {1}".format(X_train_A.shape, X_train_B.shape) print "test size: {0} {1}".format(X_test_A.shape, X_test_B.shape) #def transform_features(x): # return np.log(1+x) X_train_minus = transform_features(X_train_A) - transform_features(X_train_B) X_train_div = transform_features(X_train_A) / (transform_features(X_train_B) + 1) X_train = np.concatenate((X_train_div, X_train_minus),axis=1) X_test_minus = transform_features(X_test_A) - transform_features(X_test_B) X_test_div = transform_features(X_test_A) / (transform_features(X_test_B) + 1) X_test = np.concatenate((X_test_div, X_test_minus),axis=1) #In this case we'll use a random forest, but this could be any classifier cfr = RandomForestClassifier(n_estimators=100, max_features=math.sqrt(X_train.shape[1]), n_jobs=1) #Simple K-Fold cross validation. 5 folds. cv = cross_validation.KFold(len(X_train), k=10, indices=False) #iterate through the training and test cross validation segments and #run the classifier on each one, aggregating the results into a list results = [] for traincv, testcv in cv: probas = cfr.fit(X_train[traincv], y_train[traincv]).predict_proba(X_train[testcv]) p_train = [x[1] for x in probas] results.append(auc_score(y_train[testcv].tolist(),p_train)) #results.append( logloss.llfun(target[testcv], [x[1] for x in probas]) ) #print out the mean of the cross-validated results print "Results: " + str( np.array(results).mean() ) # Test set prob probas = cfr.predict_proba(X_test) p_test = [x[1] for x in probas] ########################### # WRITING SUBMISSION FILE ########################### predfile = open('predictions_test.csv','w+') print "label size: test - {0} expected {1}".format(len(p_test), X_test_A.shape[0]) for item in p_train: print >>predfile, "{0}".format(str(item)) predfile.close()
def AUROCScore(self): try: self.__rocarea = auc_score(self.__labels, self.__scores) except Exception as e: print "roc_curve exception" print e return nan return self.__rocarea
def test_roc_curve(): """Test Area under Receiver Operating Characteristic (ROC) curve""" y_true, _, probas_pred = make_prediction(binary=True) fpr, tpr, thresholds = roc_curve(y_true, probas_pred) roc_auc = auc(fpr, tpr) assert_array_almost_equal(roc_auc, 0.80, decimal=2) assert_almost_equal(roc_auc, auc_score(y_true, probas_pred))
def summary(clf, x, y): df = clf.decision_function(x).ravel() yp = df > 0 print 'False Positive: %0.3f' % false_pos(y, yp) print 'Recall: %0.3f' % recall(y, yp) print 'AUC: %0.3f' % auc_score(y, yp) print 'Accuracy: %0.3f' % (yp == y).mean()
def iterate_Multinomial_alpha(vect): auc_training=[] auc_oos=[] dfs=np.arange(0,3,0.1) print dfs for n in dfs: print n train2=vect.fit_transform(train.Comment) x_train2,x_test2=train_test_split(train2,random_state=42) x_train2=x_train2.tocoo() x_test2=x_test2.tocoo() classifier = MultinomialNB(fit_prior=True, alpha=n).fit(x_train2,x_train[:,0]) auc_training.append(auc_score(x_train[:,0],classifier.predict(x_train2))) auc_oos.append(auc_score(x_test[:,0],classifier.predict(x_test2))) results= zip(dfs,auc_training,auc_oos) print results return auc_plot(results)
def validation_worker(args): X, y, model, j, SEED = args X_train, X_cv, y_train, y_cv = cross_validation.train_test_split( X, y, test_size=.15, random_state = j*SEED) model.fit(X_train, y_train) preds = model.predict_proba(X_cv)[:,1] auc = metrics.auc_score(y_cv, preds) return auc
def cv_loop(X, y, model, N, seed): mean_auc = 0.0 k_fold = KFold(len(y), N, indices=True, shuffle=True, random_state=seed) for train_ix, test_ix in k_fold: model.fit(X[train_ix], y[train_ix]) preds = model.predict_proba(X[test_ix])[:, 1] auc = metrics.auc_score(y[test_ix], preds) # print("AUC (fold %d/%d): %f" % (i + 1, N, auc)) mean_auc += auc return mean_auc / N
def cv_loop(X, y, model, N): mean_auc = 0.0 for i in range(N): X_train, X_cv, y_train, y_cv = cross_validation.train_test_split(X, y, test_size=0.20, random_state=i * SEED) model.fit(X_train, y_train) preds = model.predict_proba(X_cv)[:, 1] auc = metrics.auc_score(y_cv, preds) print "AUC (fold %d/%d): %f" % (i + 1, N, auc) mean_auc += auc return mean_auc / N
def iterate_vectorizer(initial_value,iterations,vect_string): auc_training=[] auc_oos=[] dfs=range(initial_value,iterations) print dfs for n in dfs: print n vect = eval(vect_string % n) train2=vect.fit_transform(train.Comment) x_train2,x_test2=train_test_split(train2,random_state=42) x_train2=x_train2.tocoo() x_test2=x_test2.tocoo() classifier = MultinomialNB(fit_prior=True).fit(x_train2,x_train[:,0]) #the third term has to be a list, or an array) auc_training.append(auc_score(x_train[:,0],classifier.predict(x_train2))) auc_oos.append(auc_score(x_test[:,0],classifier.predict(x_test2))) results= zip(dfs,auc_training,auc_oos) print results return auc_plot(results) return results
def calculate_classification_statistics(dataset): # Old format: # 'Expected true and predicted labels for each fold, but failed.' + # 'If you wish to provide labels for each fold separately it should look like: ' + # '[[y_true_1, y_predicted_1], [y_true_2, y_predicted_2], ...]') labels = [[],[]] for i in range(0,len(dataset.target)): labels[0].append(dataset['target'][i]) labels[1].append(dataset['targetPredicted'][i]) # Check if we have true and predicted labels for each fold if labels and type(labels[0][0]) == list: try: # Flatten y_true, y_pred = [], [] for fold_labels in labels: y_true.extend(fold_labels[0]) y_pred.extend(fold_labels[1]) labels = [y_true, y_pred] except: raise Exception('Expected true and predicted labels for each fold, but failed.' + 'If you wish to provide labels for each fold separately it should look like: ' + '[[y_true_1, y_predicted_1], [y_true_2, y_predicted_2], ...]') if len(labels) != 2: raise Exception('Wrong input structure, this widget accepts labels in the form: [y_true, y_pred]') y_true, y_pred = labels classes = set() classes.update(y_true + y_pred) classes = sorted(list(classes)) # Assign integers to classes class_to_int = {} for i, cls_label in enumerate(classes): class_to_int[cls_label] = i y_true = [class_to_int[lbl] for lbl in y_true] y_pred = [class_to_int[lbl] for lbl in y_pred] accuracy = metrics.accuracy_score(y_true, y_pred) precision = metrics.precision_score(y_true, y_pred) recall = metrics.recall_score(y_true, y_pred) f1 = metrics.f1_score(y_true, y_pred) confusion_matrix = metrics.confusion_matrix(y_true, y_pred) # AUC is defined only for binary classes if len(classes) == 2: auc = metrics.auc_score(y_true, y_pred) else: auc = 'undefined for multiple classes' return accuracy, precision, recall, f1, auc, confusion_matrix
def r_funct(current_key,str_values): df_train = pd.DataFrame.from_records(str_values,columns = df_columns) ''' #0. convert to proper dtypes for col,coltype in data_type_dict.iteritems(): if coltype=='int64': df_train[col] = df_train[col].astype(int) if coltype=='float64': df_train[col] = df_train[col].astype(float) ''' #1. remove constant columns remove = [] for col in df_train.columns: if df_train[col].std() == 0: remove.append(col) df_train = df_train.drop(remove, axis=1) #2. remove duplicated columns remove = [] c = df_train.columns for i in range(len(c)-1): v = df_train[c[i]].values for j in range(i+1,len(c)): if np.array_equal(v,df_train[c[j]].values): remove.append(c[j]) df_train = df_train.drop(remove, axis=1) #REMOVE UNWANTED COLUMNS y_train = df_train['TARGET'].values X_train = df_train.drop(['ID','TARGET'], axis=1).values # params for this randomforest len_train = len(X_train) learning_rate=random.choice([1,.5,.3,.2,.1,.03,.05,.01,.005,.001,.0005,.0001,.00001]) n_estimators=sp_randint.rvs(100, 5000) subsample=random.choice([1,.95,.85,.90,.8]) min_samples_split=sp_randint.rvs(2, 11) min_samples_leaf=sp_randint.rvs(1, 11) max_depth=sp_randint.rvs(2, 20) min_weight_fraction_leaf=0 # kfold cross validation for train data using randomforest. clf = GradientBoostingClassifier(learning_rate=learning_rate,n_estimators=n_estimators,subsample=subsample,min_samples_split=min_samples_split,min_samples_leaf=min_samples_leaf,max_depth=max_depth) k_fold = cross_validation.KFold(len_train, 5) auc_scores_list = [] for k, (train, test) in enumerate(k_fold): clf.fit(X_train[train], y_train[train]) auc_scr = auc_score(y_train[test], clf.predict_proba(X_train[test])[:,1]) auc_scores_list.append(auc_scr) mean = np.mean(auc_scores_list) std = np.std(auc_scores_list) print "GBT:learning_rate:%s,n_estimators:%s,subsample:%s,min_samples_split:%s,min_samples_leaf:%s,min_weight_fraction_leaf:%s,max_depth:%s,mean:%s,std:%s" %(learning_rate,n_estimators,subsample,min_samples_split,min_samples_leaf,min_weight_fraction_leaf,max_depth,mean,std)
def demo(): # Basic demo test X_train, y_train = load_svmlight_file("demo.train", query_id=False) X_test, y_test = load_svmlight_file("demo.train", query_id=False) coef, _ = sgd_train( X_train, y_train, np.ones(y_test.shape), alpha=0.1, n_features=150000, model="rank", max_iter=100000 ) preds = sgd_predict(X_test, coef, blocks=None) preds = np.sign(preds) assert accuracy_score(y_test, preds) > 0.98 assert precision_score(y_test, preds) > 0.98 assert recall_score(y_test, preds) > 0.98 assert auc_score(y_test, preds) > 0.98
def auc(y, y_pred): #y_1 = y[(y['Target'] == 0) | (y['Target'] == 1)] y_1 = y.copy() y_1[y.Target == 0] = -1 #print y_1 #predictions = y_p.ix[y_1.index] auc1 = metrics.auc_score(np.array(y_1.Target), y_pred) #y_2 = y[(y['Target'] == 0) | (y['Target'] == -1)] #predictions = y_p.ix[y_2.index] #predictions = predictions * (-1) y_2 = y.copy() y_2[y.Target == 0] = 1 #y_2 = y_2.replace(-1, 1) #print predictions.shape, y_2.shape #print len(y_1),len(y_2.Target) auc2 = metrics.auc_score(np.array(y_2.Target), y_pred) auc = 0.5 * auc1 + 0.5 * auc2 #print "AUC", auc, auc1, auc2 return auc, auc1, auc2
def cv_loop_mt(args): f, X, y, model, N = args sum_auc = 0. for i in range(N): X_train, X_cv, y_train, y_cv = cross_validation.train_test_split( X, y, test_size=.20, random_state = i*SEED) model.fit(X_train, y_train) preds = model.predict_proba(X_cv)[:,1] auc = metrics.auc_score(y_cv, preds) print 'Feature Set {f} AUC (fold {current}/{total}): {auc}'.format(f = f, current = i + 1, total = N, auc = auc) sum_auc += auc mean_auc = sum_auc / N return (mean_auc, f)
def report(self): from sklearn.metrics import auc_score from sklearn.metrics import classification_report from sklearn.metrics import confusion_matrix from sklearn.metrics import f1_score from sklearn.metrics import precision_recall_curve y_pred_probas, y_true, md = self.make_predictions() y_pred = y_pred_probas.argmax(1) y_pred_probas = y_pred_probas[:, 1] y_true = y_true.reshape(-1) print print "AUC score:", auc_score(y_true, y_pred_probas) print "AUC score (binary):", auc_score(y_true, y_pred) print print "Classification report:" print classification_report(y_true, y_pred) print print "Confusion matrix:" print confusion_matrix(y_true, y_pred) print
def cv_loop(X, y, model, N): mean_auc = 0. for i in range(N): X_train, X_cv, y_train, y_cv = cross_validation.train_test_split( X, y, test_size=.10, random_state = i*SEED) model.fit(X_train, y_train) preds = model.predict_proba(X_cv) pVec = np.reshape(preds, 19 * preds.shape[0]) yVec = np.reshape(y_cv, 19 * y_cv.shape[0]) auc = metrics.auc_score(yVec, pVec) print "AUC (fold %d/%d): %f" % (i + 1, N, auc) mean_auc += auc return mean_auc/N
def perfeval_classification_statistics(input_dict): from sklearn import metrics labels = input_dict['true_and_predicted_labels'] pos_label = input_dict.get('pos_label', None) # Check if we have true and predicted labels for each fold if labels and type(labels[0][0]) == list: try: # Flatten y_true, y_pred = [], [] for fold_labels in labels: y_true.extend(fold_labels[0]) y_pred.extend(fold_labels[1]) labels = [y_true, y_pred] except: raise Exception('Expected true and predicted labels for each fold, but failed.' + 'If you wish to provide labels for each fold separately it should look like: ' + '[[y_true_1, y_predicted_1], [y_true_2, y_predicted_2], ...]') if len(labels) != 2: raise Exception('Wrong input structure, this widget accepts labels in the form: [y_true, y_pred]') y_true, y_pred = labels classes = set() classes.update(y_true + y_pred) classes = sorted(list(classes)) # Assign integers to classes class_to_int = {} for i, cls_label in enumerate(classes): class_to_int[cls_label] = i y_true = [class_to_int[lbl] for lbl in y_true] y_pred = [class_to_int[lbl] for lbl in y_pred] accuracy = metrics.accuracy_score(y_true, y_pred) precision = metrics.precision_score(y_true, y_pred) recall = metrics.recall_score(y_true, y_pred) f1 = metrics.f1_score(y_true, y_pred) confusion_matrix = metrics.confusion_matrix(y_true, y_pred) # AUC is defined only for binary classes if len(classes) == 2: auc = metrics.auc_score(y_true, y_pred) else: auc = 'undefined for multiple classes' return {'accuracy': accuracy, 'precision': precision, 'recall': recall, 'f1': f1, 'auc': auc, 'confusion_matrix': confusion_matrix}
def inv_auc_score(y_true, y_scores): return 1.0 - metrics.auc_score(y_true, y_scores)
for e in xrange(epochs): print('\nEpoch {:d}/{:d}'.format(e + 1, epochs)) print('Learning rate: {:6f}'.format(K.eval(ae.optimizer.lr))) curr_iter = 0 train_loss = [] for batch_adj, batch_train, dummy_f, dummy_y, dummy_m in batch_data: # Each iteration/loop is a batch of train_batch_size samples res = ae.train_on_batch([batch_adj], [batch_train]) train_loss.append(res) curr_iter += 1 if curr_iter >= num_iters_per_train_epoch: break train_loss = np.asarray(train_loss) train_loss = np.mean(train_loss, axis=0) print('Avg. training loss: {:6f}'.format(train_loss)) print('\nEvaluating val set...') decoded_lp = np.empty(shape=adj.shape, dtype=np.float32) predictions = [] for step in xrange(adj.shape[0] / val_batch_size + 1): low = step * val_batch_size high = low + val_batch_size batch_adj = adj[low:high].toarray() if batch_adj.shape[0] == 0: break decoded_lp[low:high] = ae.predict_on_batch([batch_adj]) predictions.extend(decoded_lp[test_r, test_c]) predictions.extend(decoded_lp[test_c, test_r]) print('Val AUC: {:6f}'.format(auc_score(labels, predictions))) print('Val AP: {:6f}'.format(ap_score(labels, predictions))) print('\nAll Done.')
def target_score(y_true, predictions): return (auc_score(y_true == 1, predictions) + auc_score(y_true == -1, -predictions)) / 2
def GetErrorPrimal(data, K, w, b, giveMCC=True, givePercentError=True, giveAUC=True): predictions = [] ys = data[:, 0].reshape(1, len(data[:, 0])) predictions = b + np.dot(K, w) ''' import matplotlib.pyplot as pl pl.subplot(1,3,1) pl.title("Original space") pl.plot([data[i,1] for i in range(len(predictions)) if data[i,0]<0], [data[i,2] for i in range(len(predictions)) if data[i,0]<0], "ro") pl.plot([data[i,1] for i in range(len(predictions)) if data[i,0]>0], [data[i,2] for i in range(len(predictions)) if data[i,0]>0], "bo") pl.xlabel("x") pl.ylabel("y") pl.subplot(1,3,2) pl.title("Kernel space predicted") pl.plot([K[i,0] for i in range(len(predictions)) if predictions[i]<0], [K[i,1] for i in range(len(predictions)) if predictions[i]<0], "ro") pl.plot([K[i,0] for i in range(len(predictions)) if predictions[i]>0], [K[i,1] for i in range(len(predictions)) if predictions[i]>0], "bo") pl.xlabel("first component") pl.ylabel("second component") pl.subplot(1,3,3) pl.title("Kernel space actual") pl.plot([K[i,0] for i in range(len(predictions)) if data[i,0]<0], [K[i,1] for i in range(len(predictions)) if data[i,0]<0], "ro") pl.plot([K[i,0] for i in range(len(predictions)) if data[i,0]>0], [K[i,1] for i in range(len(predictions)) if data[i,0]>0], "bo") pl.xlabel("first component") pl.ylabel("second component") pl.show() ''' if giveMCC == True: temp = doMCC(predictions, data) MCC = round(temp[0], 2) percentError = round(temp[1], 4) results = temp[2] if giveMCC == False: temp = ne.evaluate('ys*predictions') percentError = round( .5 * (len(predictions) - ne.evaluate('sum(temp)')) / float(len(predictions)), 4) if giveAUC == True: AUC = round(sm.auc_score(data[:, 0], np.array(predictions)), 2) if giveMCC == True and giveAUC == True: return { 'MCC': MCC, 'results': results, '%E': percentError, 'AUC': AUC, 'predictons': None } if giveMCC == True and giveAUC == False: return { 'MCC': MCC, 'results': results, '%E': percentError, 'predictons': None } if giveMCC == False and giveAUC == False: return {'AUC': None, '%E': percentError}
def GetErrorDual(InSampleData, OutSampleData, K, Kdecomp, a, b, giveMCC=True, givePercentError=True, giveAUC=True): H = OutSampleData[:, 0].size predictions = [] y_in = InSampleData[:, 0].reshape(len(InSampleData[:, 0]), 1) y_out = OutSampleData[:, 0].reshape(1, len(OutSampleData[:, 0])) z = ne.evaluate('a*y_in') for i in xrange(H): X = b + np.dot(K[i], z) predictions.append(X) ''' import matplotlib.pyplot as pl data=OutSampleData K=Kdecomp pl.subplot(1,3,1) pl.title("Original space") pl.plot([data[i,1] for i in range(len(predictions)) if data[i,0]<0], [data[i,2] for i in range(len(predictions)) if data[i,0]<0], "ro") pl.plot([data[i,1] for i in range(len(predictions)) if data[i,0]>0], [data[i,2] for i in range(len(predictions)) if data[i,0]>0], "bo") pl.xlabel("x") pl.ylabel("y") pl.subplot(1,3,2) pl.title("Kernel space with guessed-predictions") pl.plot([K[i,0] for i in range(len(predictions)) if predictions[i]<0], [K[i,1] for i in range(len(predictions)) if predictions[i]<0], "ro") pl.plot([K[i,0] for i in range(len(predictions)) if predictions[i]>0], [K[i,1] for i in range(len(predictions)) if predictions[i]>0], "bo") pl.xlabel("first component") pl.ylabel("second component") pl.subplot(1,3,3) pl.title("Kernel space with true-predictions") pl.plot([K[i,0] for i in range(len(predictions)) if data[i,0]<0], [K[i,1] for i in range(len(predictions)) if data[i,0]<0], "ro") pl.plot([K[i,0] for i in range(len(predictions)) if data[i,0]>0], [K[i,1] for i in range(len(predictions)) if data[i,0]>0], "bo") pl.xlabel("first component") pl.ylabel("second component") pl.show() ''' if giveMCC == True: temp = doMCC(predictions, OutSampleData) MCC = round(temp[0], 2) percentError = round(temp[1], 4) results = temp[2] if giveMCC == False: temp = ne.evaluate('y_out*predictions') percentError = round( .5 * (len(predictions) - ne.evaluate('sum(temp)')) / float(len(predictions)), 4) if giveAUC == True: AUC = round(sm.auc_score(OutSampleData[:, 0], np.array(predictions)), 2) if giveMCC == True and giveAUC == True: return { 'MCC': MCC, 'results': results, '%E': percentError, 'AUC': AUC, 'predictons': None } if giveMCC == True and giveAUC == False: return { 'MCC': MCC, 'results': results, '%E': percentError, 'predictons': None } if giveMCC == False and giveAUC == False: return {'AUC': None, '%E': percentError}
def evaluation(y_true, y_pred): return metrics.auc_score(y_true, y_pred[:, 1])
def get_auc(y, y_pred_proba): score = auc_score(y, y_pred_proba) print score