def measure(clf_class, parameters, name, data_size=None, plot=False):
start_time_clf = time.time()
if data_size is None:
X = qa_X
Y = qa_Y
else:
X = qa_X[:data_size]
Y = qa_Y[:data_size]
# cv = KFold(n=len(X), n_folds=10, indices=True)
# cv = KFold(n=len(X), n_folds=10)
# cv = KFold(n_splits=10, shuffle=True)
cv = KFold(n=len(X), n_folds=10, shuffle=True)
train_errors = []
test_errors = []
scores = []
roc_scores = []
fprs, tprs = [], []
pr_scores = []
precisions, recalls, thresholds = [], [], []
for train, test in cv:
X_train, y_train = X[train], Y[train]
X_test, y_test = X[test], Y[test]
clf = clf_class(**parameters)
clf.fit(X_train, y_train)
train_score = clf.score(X_train, y_train)
test_score = clf.score(X_test, y_test)
train_errors.append(1 - train_score)
test_errors.append(1 - test_score)
scores.append(test_score)
proba = clf.predict_proba(X_test)
label_idx = 1
fpr, tpr, roc_thresholds = roc_curve(y_test, proba[:, label_idx])
precision, recall, pr_thresholds = precision_recall_curve(
y_test, proba[:, label_idx])
roc_scores.append(auc(fpr, tpr))
fprs.append(fpr)
tprs.append(tpr)
pr_scores.append(auc(recall, precision))
precisions.append(precision)
recalls.append(recall)
thresholds.append(pr_thresholds)
print(
classification_report(y_test,
proba[:, label_idx] > 0.63,
target_names=['not accepted', 'accepted']))
# get medium clone
scores_to_sort = pr_scores # roc_scores
medium = np.argsort(scores_to_sort)[int(len(scores_to_sort) / 2)]
if plot:
# plot_roc(roc_scores[medium], name, fprs[medium], tprs[medium])
plot_pr(pr_scores[medium], name, precisions[medium], recalls[medium],
classifying_answer + " answers")
if hasattr(clf, 'coef_'):
plot_feat_importance(feature_names, clf, name)
summary = (name, np.mean(scores), np.std(scores), np.mean(roc_scores),
np.std(roc_scores), np.mean(pr_scores), np.std(pr_scores),
time.time() - start_time_clf)
print(summary)
avg_scores_summary.append(summary)
precisions = precisions[medium]
recalls = recalls[medium]
thresholds = np.hstack(([0], thresholds[medium]))
idx80 = precisions >= 0.8
print("P=%.2f R=%.2f thresh=%.2f" %
(precisions[idx80][0], recalls[idx80][0], thresholds[idx80][0]))
return np.mean(train_errors), np.mean(test_errors)
def measure(clf_class, parameters, name, data_size=None, plot=False):
start_time_clf = time.time()
if data_size is None:
X = qa_X
Y = qa_Y
else:
X = qa_X[:data_size]
Y = qa_Y[:data_size]
cv = KFold(n=len(X), n_folds=10, indices=True)
train_errors = []
test_errors = []
scores = []
roc_scores = []
fprs, tprs = [], []
pr_scores = []
precisions, recalls, thresholds = [], [], []
for train, test in cv:
X_train, y_train = X[train], Y[train]
X_test, y_test = X[test], Y[test]
clf = clf_class(**parameters)
clf.fit(X_train, y_train)
train_score = clf.score(X_train, y_train)
test_score = clf.score(X_test, y_test)
train_errors.append(1 - train_score)
test_errors.append(1 - test_score)
scores.append(test_score)
proba = clf.predict_proba(X_test)
label_idx = 1
fpr, tpr, roc_thresholds = roc_curve(y_test, proba[:, label_idx])
precision, recall, pr_thresholds = precision_recall_curve(
y_test, proba[:, label_idx])
roc_scores.append(auc(fpr, tpr))
fprs.append(fpr)
tprs.append(tpr)
pr_scores.append(auc(recall, precision))
precisions.append(precision)
recalls.append(recall)
thresholds.append(pr_thresholds)
print(classification_report(y_test, proba[:, label_idx] >
0.63, target_names=['not accepted', 'accepted']))
# get medium clone
scores_to_sort = pr_scores # roc_scores
medium = np.argsort(scores_to_sort)[len(scores_to_sort) / 2]
if plot:
#plot_roc(roc_scores[medium], name, fprs[medium], tprs[medium])
plot_pr(pr_scores[medium], name, precisions[medium],
recalls[medium], classifying_answer + " answers")
if hasattr(clf, 'coef_'):
plot_feat_importance(feature_names, clf, name)
summary = (name,
np.mean(scores), np.std(scores),
np.mean(roc_scores), np.std(roc_scores),
np.mean(pr_scores), np.std(pr_scores),
time.time() - start_time_clf)
print(summary)
avg_scores_summary.append(summary)
precisions = precisions[medium]
recalls = recalls[medium]
thresholds = np.hstack(([0], thresholds[medium]))
idx80 = precisions >= 0.8
print("P=%.2f R=%.2f thresh=%.2f" % (precisions[idx80][0], recalls[
idx80][0], thresholds[idx80][0]))
return np.mean(train_errors), np.mean(test_errors)
def measure(clf_class, parameters, name, data_size=None, plot=False):
start_time_clf = time.time()
if data_size is None:
X = method_name()
Y = qa_Y
else:
X = qa_X[:data_size]
Y = qa_Y[:data_size]
good_job = KFold(n=len(X), n_folds=10, indices=True)
cv = good_job
l = RANGE
train_errors = []
test_errors = []
scores = []
roc_scores = []
fprs, tprs = [], []
pr_scores = []
precisions, recalls, thresholds = [], [], []
for fold_idx, (train, test) in enumerate(cv):
X_train, y_train = X[train], Y[train]
X_test, y_test = X[test], Y[test]
only_one_class_in_train = len(set(y_train)) == 1
only_one_class_in_test = len(set(y_test)) == 1
if only_one_class_in_train or only_one_class_in_test:
# this would pose problems later on
continue
clf = clf_class(**parameters)
clf.fit(X_train, y_train)
train_score = clf.score(X_train, y_train)
test_score = clf.score(X_test, y_test)
train_errors.append(1 - train_score)
test_errors.append(1 - test_score)
scores.append(test_score)
proba = clf.predict_proba(X_test)
label_idx = 1
fpr, tpr, roc_thresholds = roc_curve(y_test, proba[:, label_idx])
precision, recall, pr_thresholds = precision_recall_curve(
y_test, proba[:, label_idx])
roc_scores.append(auc(fpr, tpr))
fprs.append(fpr)
tprs.append(tpr)
pr_scores.append(auc(recall, precision))
precisions.append(precision)
recalls.append(recall)
thresholds.append(pr_thresholds)
# This threshold is determined at the end of the chapter 5,
# where we find conditions such that precision is in the area of
# about 80%. With it we trade off recall for precision.
threshold_for_detecting_good_answers = 0.59
print("Clone #%i" % fold_idx)
print(classification_report(y_test, proba[:, label_idx] >
threshold_for_detecting_good_answers, target_names=['not accepted', 'accepted']))
# get medium clone
scores_to_sort = pr_scores # roc_scores
medium = np.argsort(scores_to_sort)[len(scores_to_sort) / 2]
print("Medium clone is #%i" % medium)
if plot:
#plot_roc(roc_scores[medium], name, fprs[medium], tprs[medium])
plot_pr(pr_scores[medium], name, precisions[medium],
recalls[medium], classifying_answer + " answers")
if hasattr(clf, 'coef_'):
plot_feat_importance(feature_names, clf, name)
summary = (name,
np.mean(scores), np.std(scores),
np.mean(roc_scores), np.std(roc_scores),
np.mean(pr_scores), np.std(pr_scores),
time.time() - start_time_clf)
print(summary)
avg_scores_summary.append(summary)
precisions = precisions[medium]
recalls = recalls[medium]
thresholds = np.hstack(([0], thresholds[medium]))
idx80 = precisions >= 0.8
print("P=%.2f R=%.2f thresh=%.2f" % (precisions[idx80][0], recalls[
idx80][0], thresholds[idx80][0]))
return np.mean(train_errors), np.mean(test_errors)
def measure(clf_class, parameters, name, data_size=None, plot=False):
start_time_clf = time.time()
if data_size is None:
X = qa_X
Y = qa_Y
else:
X = qa_X[:data_size]
Y = qa_Y[:data_size]
cv = KFold(n=len(X), n_folds=10, indices=True)
train_errors = []
test_errors = []
scores = []
roc_scores = []
fprs, tprs = [], []
pr_scores = []
precisions, recalls, thresholds = [], [], []
for fold_idx, (train, test) in enumerate(cv):
X_train, y_train = X[train], Y[train]
X_test, y_test = X[test], Y[test]
only_one_class_in_train = len(set(y_train)) == 1
only_one_class_in_test = len(set(y_test)) == 1
if only_one_class_in_train or only_one_class_in_test:
# this would pose problems later on
continue
clf = clf_class(**parameters)
clf.fit(X_train, y_train)
train_score = clf.score(X_train, y_train)
test_score = clf.score(X_test, y_test)
train_errors.append(1 - train_score)
test_errors.append(1 - test_score)
scores.append(test_score)
proba = clf.predict_proba(X_test)
label_idx = 1
fpr, tpr, roc_thresholds = roc_curve(y_test, proba[:, label_idx])
precision, recall, pr_thresholds = precision_recall_curve(
y_test, proba[:, label_idx])
roc_scores.append(auc(fpr, tpr))
fprs.append(fpr)
tprs.append(tpr)
pr_scores.append(auc(recall, precision))
precisions.append(precision)
recalls.append(recall)
thresholds.append(pr_thresholds)
# This threshold is determined at the end of the chapter 5,
# where we find conditions such that precision is in the area of
# about 80%. With it we trade off recall for precision.
threshold_for_detecting_good_answers = 0.59
print("Clone #%i" % fold_idx)
print(
classification_report(
y_test,
proba[:, label_idx] > threshold_for_detecting_good_answers,
target_names=['not accepted', 'accepted']))
# get medium clone
scores_to_sort = pr_scores # roc_scores
medium = np.argsort(scores_to_sort)[len(scores_to_sort) / 2]
print("Medium clone is #%i" % medium)
if plot:
#plot_roc(roc_scores[medium], name, fprs[medium], tprs[medium])
plot_pr(pr_scores[medium], name, precisions[medium], recalls[medium],
classifying_answer + " answers")
if hasattr(clf, 'coef_'):
plot_feat_importance(feature_names, clf, name)
summary = (name, np.mean(scores), np.std(scores), np.mean(roc_scores),
np.std(roc_scores), np.mean(pr_scores), np.std(pr_scores),
time.time() - start_time_clf)
print(summary)
avg_scores_summary.append(summary)
precisions = precisions[medium]
recalls = recalls[medium]
thresholds = np.hstack(([0], thresholds[medium]))
idx80 = precisions >= 0.8
print("P=%.2f R=%.2f thresh=%.2f" %
(precisions[idx80][0], recalls[idx80][0], thresholds[idx80][0]))
return np.mean(train_errors), np.mean(test_errors)
def measure(clf_class, parameters, name, data_size=None, plot=False):
start_time_clf = time.time()
if data_size is None:
X = qa_X
Y = qa_Y
else:
X = qa_X[:data_size]
Y = qa_Y[:data_size]
cv = KFold(n=len(X), n_folds=10, indices=True)
train_errors = []
test_errors = []
scores = []
roc_scores = []
fprs, tprs = [], []
pr_scores = []
precisions, recalls, thresholds = [], [], []
# loop through n cross validation folds
for train, test in cv:
X_train, y_train = X[train], Y[train]
X_test, y_test = X[test], Y[test]
clf = clf_class(**parameters)
# fit model
clf.fit(X_train, y_train)
# predict training set, predict test set
train_score = clf.score(X_train, y_train)
test_score = clf.score(X_test, y_test)
# save training error
train_errors.append(1 - train_score)
test_errors.append(1 - test_score)
scores.append(test_score)
proba = clf.predict_proba(X_test) # return probability test dataset
# [0.8 0.2], --> prob for 0 and 1 (average of n closest neighbors)
# calculate false prediction rate, true prediction rate
# for different threshold values (threshold values are
# calculated as proportion m/n nearest neighbors
label_idx = 1
fpr, tpr, roc_thresholds = roc_curve(y_test, proba[:, label_idx])
# calculate precision, recall, and precision thresholds
precision, recall, pr_thresholds = precision_recall_curve(y_test, proba[:, label_idx])
# store results in container variables
roc_scores.append(auc(fpr, tpr))
fprs.append(fpr)
tprs.append(tpr)
pr_scores.append(auc(recall, precision))
precisions.append(precision)
recalls.append(recall)
thresholds.append(pr_thresholds)
print "classification report (accepted -> p > 0.63)"
print(classification_report(y_test, proba[:, label_idx] >
0.63, target_names=['not accepted', 'accepted']))
# get medium clone
scores_to_sort = pr_scores # roc_scores
medium = np.argsort(scores_to_sort)[len(scores_to_sort) / 2]
if plot:
#plot_roc(roc_scores[medium], name, fprs[medium], tprs[medium])
plot_pr(pr_scores[medium], name, precisions[medium],
recalls[medium], classifying_answer + " answers")
if hasattr(clf, 'coef_'):
plot_feat_importance(feature_names, clf, name)
summary = (name,
np.mean(scores), np.std(scores),
np.mean(roc_scores), np.std(roc_scores),
np.mean(pr_scores), np.std(pr_scores),
time.time() - start_time_clf)
print "summary: name, validation_dataset_correct_prediction_mean, score_std, rocscore_mean, rocscore_std, prscore_mean, prscore_std, time"
print(summary)
avg_scores_summary.append(summary)
precisions = precisions[medium]
recalls = recalls[medium]
thresholds = np.hstack(([0], thresholds[medium]))
idx80 = precisions >= 0.8
print("P=%.2f R=%.2f thresh=%.2f" % (precisions[idx80][0], recalls[
idx80][0], thresholds[idx80][0]))
return np.mean(train_errors), np.mean(test_errors)
