def plot_model_learning_curves(self, path=''):
# Plot the learning curves of the models
k = 0
size = sum((len(mod.parameters) > 1) for mod in self.models)
fig, axes = plt.subplots(2, size, sharex='col', sharey='row')
for mod in self.models:
if (len(mod.parameters) > 1):
tempModel = mod.set_params(mod.parameters[mod.best_params])
title = mod.name + ": best Parameters"
plot_learning_curve(tempModel,
title,
self.x,
self.y,
axis=axes[0, k])
tempModel = mod.set_params(mod.parameters[mod.worst_params])
title = mod.name + ": worst Parameters"
plot_learning_curve(tempModel,
title,
self.x,
self.y,
axis=axes[1, k])
k += 1
red_patch = mpatches.Patch(color='red', label='Training score')
green_patch = mpatches.Patch(color='green',
label='Cross-validation score')
fig.legend(handles=[red_patch, green_patch])
fig.text(0.5, 0.04, 'Training Examples', ha='center')
fig.text(0.04, 0.5, 'Score', va='center', rotation='vertical')
plt.savefig(path)
plt.show()
def learningCurve(dataSet, learner, stepNum, updateTxt):
data = dataSet.load()
X = data.iloc[:, 0:-1]
Y = data.iloc[:, -1]
xTrain, xTest, yTrain, yTest = model_selection.train_test_split(
X, Y, test_size=testSize, random_state=randomState)
# xTrain, xValidate, yTrain, yValidate = model_selection.train_test_split(xTrain, yTrain,
# test_size = testSize, random_state = randomState)
# cv = model_selection.ShuffleSplit(n_splits=20, test_size=testSize,
# random_state=randomState)
print("====================================================")
print(
f"LearningCurve-{stepNum}-{dataSet.plotFileName}-{learner.plotFileName}:"
)
samplePercent = [0.05, 0.1, 0.2, 0.4, 0.7, 1.0]
title = f"{learner.plotTitle} Learning Curve on {dataSet.plotTitle}"
plt, resultStr = plot_learning_curve(learner.learner,
title,
xTrain,
yTrain,
cv=kfolds,
train_sizes=samplePercent,
ylim=(0.60, 1.01),
n_jobs=4)
plt.savefig(
f"plots/{dataSet.plotFileName}_{learner.plotFileName}_{stepNum}_LearningCurve.png",
format='png')
if updateTxt:
with open(f"plots/{dataSet.plotFileName}_{learner.plotFileName}.txt",
"a") as file:
file.write(
"====================================================\n")
file.write(
f"LearningCurve-{stepNum}-{dataSet.plotFileName}-{learner.plotFileName}:"
+ "\n")
file.write(resultStr + "\n")
if learner.plotFileName == "NeuralNet":
epochs = np.linspace(20, 500, 20)
plot_nn_learning_curve(learner.learner,
title,
xTrain,
yTrain,
cv=kfolds,
train_epochs=epochs,
testSize=testSize,
ylim=None)
plt.savefig(
f"plots/{dataSet.plotFileName}_{learner.plotFileName}_{stepNum}_NNLearningCurve.png",
format='png')
learning_rate_init=0.01,
max_iter=500,
random_state=randomState)
print(np.shape(X))
xTrain, xTest, yTrain, yTest = model_selection.train_test_split(
X, Y, test_size=testSize, random_state=randomState)
if run == 'tune':
samplePercent = [0.05, 0.1, 0.2, 0.4, 0.7, 1.0]
title = f"{learner.plotTitle} Learning Curve on {dataSet.plotTitle}"
plt, resultStr = plot_learning_curve(learner.learner,
title,
xTrain,
yTrain,
cv=kfolds,
train_sizes=samplePercent,
ylim=(0.60, 1.01),
n_jobs=4)
plt.savefig(
f"plots/{dataSet.plotFileName}_{learner.plotFileName}_{dimReducer.fileName}_{stepNum}_LearningCurve.png",
format='png')
if updateTxt:
with open(
f"plots/{dataSet.plotFileName}_{dimReducer.fileName}_{learner.plotFileName}.txt",
"a") as file:
file.write(
"====================================================\n")
file.write(
f"LearningCurve-{stepNum}-{dataSet.plotFileName}-{learner.plotFileName}:"
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score
log_reg = LogisticRegressionCV(max_iter=4000)
gauss_nb = GaussianNB()
lin_disc_analy = LinearDiscriminantAnalysis()
svc = SVC(kernel="rbf")
print("\n\n\n")
from plot_learning_curve import *
for model in (log_reg, gauss_nb, lin_disc_analy, svc):
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
ac_sc = accuracy_score(y_test, y_pred)
rc_sc = recall_score(y_test, y_pred, average="weighted")
pr_sc = precision_score(y_test, y_pred, average="weighted")
f1_sc = f1_score(y_test, y_pred, average="micro")
# results.append([i, ac_sc, rc_sc, pr_sc, f1_sc])
print("Model = ", model.__class__.__name__, "\nAccuracy = ", ac_sc,
"\nRecall = ", rc_sc, "\nPrecision = ", pr_sc, "\nF1 = ", f1_sc,
"\n\n\n")
plot_learning_curve(model, model.__class__.__name__, X, y)
# In[ ]:
from sklearn.model_selection import train_test_split
from sklearn.kernel_ridge import KernelRidge
# In[ ]:
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
# In[ ]:
clf = KernelRidge(alpha=1.0, kernel=my_kernel)
clf.fit(X_train, y_train)
# In[ ]:
train_score = clf.score(X_train, y_train)
test_score = clf.score(X_test, y_test)
print('train score: {0}; test score: {1}'.format(train_score, test_score))
# In[ ]:
from plot_learning_curve import *
title = "Learning Curves (Kernel Ridge Regression)"
cv = ShuffleSplit(n_splits=10, test_size=0.2, random_state=0)
plot_learning_curve(clf, title, X, y, cv=cv, n_jobs=4)
plt.show()
# In[ ]:
print(f"NN-ParamSearch-{stepNum}-{nn.fileName}:")
print(resultStr)
if updateTxt:
with open(f"plots/NN_{nn.fileName}.txt", "a") as file:
file.write(
"====================================================\n")
file.write(f"ParamSearch-{stepNum}-{nn.fileName}:" + "\n")
file.write(resultStr + "\n")
elif run == "learningcurve":
title = f"Learning Curve for {nn.plotTitle}"
nn.createModel()
plt, resultStr = plot_learning_curve(nn.model,
title,
xTrain,
yTrain,
cv=kfolds)
plt.savefig(f"plots/NN_{nn.fileName}_{stepNum}_LearningCurve.png",
format='png')
print(f"NN-LearningCurve-{stepNum}-{nn.fileName}:")
print(resultStr)
if updateTxt:
with open(f"plots/NN_{nn.fileName}.txt", "a") as file:
file.write(
"====================================================\n")
file.write(f"LearningCurve-{stepNum}-{nn.fileName}:" + "\n")
file.write(resultStr + "\n")
epochs = np.linspace(1e4, maxIters, 20)