def plot_model_learning_curves(self, path=''): # Plot the learning curves of the models k = 0 size = sum((len(mod.parameters) > 1) for mod in self.models) fig, axes = plt.subplots(2, size, sharex='col', sharey='row') for mod in self.models: if (len(mod.parameters) > 1): tempModel = mod.set_params(mod.parameters[mod.best_params]) title = mod.name + ": best Parameters" plot_learning_curve(tempModel, title, self.x, self.y, axis=axes[0, k]) tempModel = mod.set_params(mod.parameters[mod.worst_params]) title = mod.name + ": worst Parameters" plot_learning_curve(tempModel, title, self.x, self.y, axis=axes[1, k]) k += 1 red_patch = mpatches.Patch(color='red', label='Training score') green_patch = mpatches.Patch(color='green', label='Cross-validation score') fig.legend(handles=[red_patch, green_patch]) fig.text(0.5, 0.04, 'Training Examples', ha='center') fig.text(0.04, 0.5, 'Score', va='center', rotation='vertical') plt.savefig(path) plt.show()
def learningCurve(dataSet, learner, stepNum, updateTxt): data = dataSet.load() X = data.iloc[:, 0:-1] Y = data.iloc[:, -1] xTrain, xTest, yTrain, yTest = model_selection.train_test_split( X, Y, test_size=testSize, random_state=randomState) # xTrain, xValidate, yTrain, yValidate = model_selection.train_test_split(xTrain, yTrain, # test_size = testSize, random_state = randomState) # cv = model_selection.ShuffleSplit(n_splits=20, test_size=testSize, # random_state=randomState) print("====================================================") print( f"LearningCurve-{stepNum}-{dataSet.plotFileName}-{learner.plotFileName}:" ) samplePercent = [0.05, 0.1, 0.2, 0.4, 0.7, 1.0] title = f"{learner.plotTitle} Learning Curve on {dataSet.plotTitle}" plt, resultStr = plot_learning_curve(learner.learner, title, xTrain, yTrain, cv=kfolds, train_sizes=samplePercent, ylim=(0.60, 1.01), n_jobs=4) plt.savefig( f"plots/{dataSet.plotFileName}_{learner.plotFileName}_{stepNum}_LearningCurve.png", format='png') if updateTxt: with open(f"plots/{dataSet.plotFileName}_{learner.plotFileName}.txt", "a") as file: file.write( "====================================================\n") file.write( f"LearningCurve-{stepNum}-{dataSet.plotFileName}-{learner.plotFileName}:" + "\n") file.write(resultStr + "\n") if learner.plotFileName == "NeuralNet": epochs = np.linspace(20, 500, 20) plot_nn_learning_curve(learner.learner, title, xTrain, yTrain, cv=kfolds, train_epochs=epochs, testSize=testSize, ylim=None) plt.savefig( f"plots/{dataSet.plotFileName}_{learner.plotFileName}_{stepNum}_NNLearningCurve.png", format='png')
learning_rate_init=0.01, max_iter=500, random_state=randomState) print(np.shape(X)) xTrain, xTest, yTrain, yTest = model_selection.train_test_split( X, Y, test_size=testSize, random_state=randomState) if run == 'tune': samplePercent = [0.05, 0.1, 0.2, 0.4, 0.7, 1.0] title = f"{learner.plotTitle} Learning Curve on {dataSet.plotTitle}" plt, resultStr = plot_learning_curve(learner.learner, title, xTrain, yTrain, cv=kfolds, train_sizes=samplePercent, ylim=(0.60, 1.01), n_jobs=4) plt.savefig( f"plots/{dataSet.plotFileName}_{learner.plotFileName}_{dimReducer.fileName}_{stepNum}_LearningCurve.png", format='png') if updateTxt: with open( f"plots/{dataSet.plotFileName}_{dimReducer.fileName}_{learner.plotFileName}.txt", "a") as file: file.write( "====================================================\n") file.write( f"LearningCurve-{stepNum}-{dataSet.plotFileName}-{learner.plotFileName}:"
from sklearn.svm import SVC from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score log_reg = LogisticRegressionCV(max_iter=4000) gauss_nb = GaussianNB() lin_disc_analy = LinearDiscriminantAnalysis() svc = SVC(kernel="rbf") print("\n\n\n") from plot_learning_curve import * for model in (log_reg, gauss_nb, lin_disc_analy, svc): model.fit(X_train, y_train) y_pred = model.predict(X_test) ac_sc = accuracy_score(y_test, y_pred) rc_sc = recall_score(y_test, y_pred, average="weighted") pr_sc = precision_score(y_test, y_pred, average="weighted") f1_sc = f1_score(y_test, y_pred, average="micro") # results.append([i, ac_sc, rc_sc, pr_sc, f1_sc]) print("Model = ", model.__class__.__name__, "\nAccuracy = ", ac_sc, "\nRecall = ", rc_sc, "\nPrecision = ", pr_sc, "\nF1 = ", f1_sc, "\n\n\n") plot_learning_curve(model, model.__class__.__name__, X, y)
# In[ ]: from sklearn.model_selection import train_test_split from sklearn.kernel_ridge import KernelRidge # In[ ]: X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) # In[ ]: clf = KernelRidge(alpha=1.0, kernel=my_kernel) clf.fit(X_train, y_train) # In[ ]: train_score = clf.score(X_train, y_train) test_score = clf.score(X_test, y_test) print('train score: {0}; test score: {1}'.format(train_score, test_score)) # In[ ]: from plot_learning_curve import * title = "Learning Curves (Kernel Ridge Regression)" cv = ShuffleSplit(n_splits=10, test_size=0.2, random_state=0) plot_learning_curve(clf, title, X, y, cv=cv, n_jobs=4) plt.show() # In[ ]:
print(f"NN-ParamSearch-{stepNum}-{nn.fileName}:") print(resultStr) if updateTxt: with open(f"plots/NN_{nn.fileName}.txt", "a") as file: file.write( "====================================================\n") file.write(f"ParamSearch-{stepNum}-{nn.fileName}:" + "\n") file.write(resultStr + "\n") elif run == "learningcurve": title = f"Learning Curve for {nn.plotTitle}" nn.createModel() plt, resultStr = plot_learning_curve(nn.model, title, xTrain, yTrain, cv=kfolds) plt.savefig(f"plots/NN_{nn.fileName}_{stepNum}_LearningCurve.png", format='png') print(f"NN-LearningCurve-{stepNum}-{nn.fileName}:") print(resultStr) if updateTxt: with open(f"plots/NN_{nn.fileName}.txt", "a") as file: file.write( "====================================================\n") file.write(f"LearningCurve-{stepNum}-{nn.fileName}:" + "\n") file.write(resultStr + "\n") epochs = np.linspace(1e4, maxIters, 20)