def _kernel_regression_and_rolling_mean(self, X, y, window): """ Implementation of Nadaraya-Watson regression and rolling_mean to the time series. :param X: array-like of shape = [n_samples, n_features] The training input samples. :param y: array-like, shape = [n_samples] The target values :param window: int Size of the moving window. This is the number of observations used for calculating the statistic. :return: tuple of 3 array-like, shape = [n_samples] Predicted target values, rolling_mean target values and rolling_std of the target values """ kr = KernelRegression(kernel="rbf", gamma=np.logspace(-2, 2, 10)) y_kr = kr.fit(X, y).predict(X) y_rm = pd.rolling_mean(y_kr, window=window,axis=1) y_std = pd.rolling_std(y, window=window,axis=1) return y_kr, y_rm, y_std
############################################################################### # Fit regression models svr = GridSearchCV(SVR(kernel='rbf'), cv=5, param_grid={ "C": [1e-1, 1e0, 1e1, 1e2], "gamma": np.logspace(-2, 2, 10) }) kr = KernelRegression(kernel="rbf", gamma=np.logspace(-2, 2, 10)) t0 = time.time() y_svr = svr.fit(X, y).predict(X) print "SVR complexity and bandwidth selected and model fitted in %.3f s" \ % (time.time() - t0) t0 = time.time() y_kr = kr.fit(X, y).predict(X) print "KR including bandwith fitted in %.3f s" \ % (time.time() - t0) ############################################################################### # Visualize models plt.scatter(X, y, c='k', label='data') plt.hold('on') plt.plot(X, y_kr, c='g', label='Kernel Regression') plt.plot(X, y_svr, c='r', label='SVR') plt.xlabel('data') plt.ylabel('target') plt.title('Kernel regression versus SVR') plt.legend() # Visualize learning curves
if AUTOSCALE: scalefactor = max((1.0, obsdf.max().max(), fcdf.max().max())) else: scalefactor = 1.0 scalefactors[node] = scalefactor outmeandict[node] = {} outvardict[node] = {} # Pull out each node's DF's seperately. for (k1, fcc), (k2, obsc) in izip(fcdf.iteritems(), obsdf.iteritems()): # print k1 obsc = obsc.dropna()/scalefactor fcc = fcc.ix[obsc.index]/scalefactor # Kernelregression Forecasted => mean kreg.fit(fcc.values.reshape(-1, 1), obsc) meanpredict = kreg.predict(testx.reshape(-1, 1)) # Select optimal bandwidth # kreg.gamma = kreg._optimize_gamma(gammas) # Our predicted mean based on point forecast # prediction = np.polyval(meanpolycoeff, fcc) prediction = np.interp(fcc, testx, meanpredict) # Calculate errors squared err2 = (prediction - obsc)**2 # Fit variance curve kreg.fit(fcc.values.reshape(-1, 1), err2) varpredict = kreg.predict(testx.reshape(-1, 1)) # Select optimal bandwidth # kreg.gamma = kreg._optimize_gamma(gammas) # Save the coefficients of the polynomial fit.
############################################################################### # Add noise to targets y += 0.5 * (0.5 - np.random.rand(y.size)) ############################################################################### # Fit regression models svr = GridSearchCV(SVR(kernel='rbf'), cv=5, param_grid={"C": [1e-1, 1e0, 1e1, 1e2], "gamma": np.logspace(-2, 2, 10)}) kr = KernelRegression(kernel="rbf", gamma=np.logspace(-2, 2, 10)) t0 = time.time() y_svr = svr.fit(X, y).predict(X) print("SVR complexity and bandwidth selected and model fitted in %.3f s" \ % (time.time() - t0)) t0 = time.time() y_kr = kr.fit(X, y).predict(X) print("KR including bandwith fitted in %.3f s" \ % (time.time() - t0)) ############################################################################### # Visualize models plt.scatter(X, y, c='k', label='data') plt.hold('on') plt.plot(X, y_kr, c='g', label='Kernel Regression') plt.plot(X, y_svr, c='r', label='SVR') plt.xlabel('data') plt.ylabel('target') plt.title('Kernel regression versus SVR') plt.legend() # Visualize learning curves
import pandas as pd from statsmodels.nonparametric.api import KernelReg from kernel_regression import KernelRegression from sklearn.cross_validation import train_test_split as sk_split import numpy as np df = pd.read_csv("abalone.data", header=None) print df.shape X = df.loc[:, 1:7].as_matrix() y = df.loc[:, 8].as_matrix().reshape(-1, 1) print X.shape print y.shape X_train, X_test, y_train, y_test = sk_split(X, y, test_size=0.20) kr = KernelRegression(kernel="rbf") kr.fit(X_train, y_train) print len(X_test) # Memory issues split X_test, just did two chunks here X_test_1 = X_test[0:100, :] X_test_2 = X_test[101:200, :] pred_y = kr.predict(X_test_1) pred_y = kr.predict(X_test_2)
def plot_regression(): np.random.seed(12345) fig, axes = plt.subplots(4, 4) for i, ax in enumerate(axes.flatten()): n_in = 1 n_out = 1 d = np.random.randint(1, 5) n_ex = np.random.randint(5, 500) std = np.random.randint(0, 1000) intercept = np.random.rand() * np.random.randint(-300, 300) X_train, y_train, X_test, y_test, coefs = random_regression_problem( n_ex, n_in, n_out, d=d, intercept=intercept, std=std, seed=i) LR = LinearRegression(fit_intercept=True) LR.fit(X_train, y_train) y_pred = LR.predict(X_test) loss = np.mean((y_test.flatten() - y_pred.flatten())**2) d = 3 best_loss = np.inf for gamma in np.linspace(1e-10, 1, 100): for c0 in np.linspace(-1, 1000, 100): kernel = "PolynomialKernel(d={}, gamma={}, c0={})".format( d, gamma, c0) KR_poly = KernelRegression(kernel=kernel) KR_poly.fit(X_train, y_train) y_pred_poly = KR_poly.predict(X_test) loss_poly = np.mean( (y_test.flatten() - y_pred_poly.flatten())**2) if loss_poly <= best_loss: KR_poly_best = kernel best_loss = loss_poly print("Best kernel: {} || loss: {:.4f}".format(KR_poly_best, best_loss)) KR_poly = KernelRegression(kernel=KR_poly_best) KR_poly.fit(X_train, y_train) KR_rbf = KernelRegression(kernel="RBFKernel(gamma=0.01)") KR_rbf.fit(X_train, y_train) y_pred_rbf = KR_rbf.predict(X_test) loss_rbf = np.mean((y_test.flatten() - y_pred_rbf.flatten())**2) xmin = min(X_test) - 0.1 * (max(X_test) - min(X_test)) xmax = max(X_test) + 0.1 * (max(X_test) - min(X_test)) X_plot = np.linspace(xmin, xmax, 100) y_plot = LR.predict(X_plot) y_plot_poly = KR_poly.predict(X_plot) y_plot_rbf = KR_rbf.predict(X_plot) ax.scatter(X_test, y_test, alpha=0.5) ax.plot(X_plot, y_plot, label="OLS", alpha=0.5) ax.plot(X_plot, y_plot_poly, label="KR (poly kernel, d={})".format(d), alpha=0.5) ax.plot(X_plot, y_plot_rbf, label="KR (rbf kernel)", alpha=0.5) ax.legend() # ax.set_title( # "MSE\nLR: {:.2f} KR (poly): {:.2f}\nKR (rbf): {:.2f}".format( # loss, loss_poly, loss_rbf # ) # ) ax.xaxis.set_ticklabels([]) ax.yaxis.set_ticklabels([]) plt.tight_layout() plt.savefig("img/kr_plots.png", dpi=300) plt.close("all")
def multivar_regress(self): # X, y = self.regression_data() X, y = self.regression_data_split() X = np.array(X) y = np.array(y) pb = X[:, 0].argsort() Xb = X[pb] yb = y[pb] X1 = np.delete(X, 1, 1) p1 = X1[:, 0].argsort() X1 = X1[p1] y1 = y[p1] X2 = np.delete(X, 0, 1) p2 = X2[:, 0].argsort() X2 = X2[p2] y2 = y[p2] x_range = np.arange(0, 0.025, 0.001) # generate a mesh y_range = np.arange(0, 1.3, 0.02) x_surf, y_surf = np.meshgrid(x_range, y_range) Xpred = np.stack((x_surf.flatten(), y_surf.flatten()), axis=1) svr = GridSearchCV(SVR(kernel='rbf'), cv=5, param_grid={ "C": [1e-1, 1e0, 1e1, 1e2], "gamma": np.logspace(-2, 2, 10) }) kr = KernelRegression(kernel="rbf", gamma=np.logspace(-2, 2, 10)) t0 = time.time() y_svrb = svr.fit(Xb, yb).predict(Xpred) print( "SVR complexity and bandwidth selected and model fitted in %.3f s" % (time.time() - t0)) score_svr = svr.score(Xb, yb) y_svr1 = svr.fit(X1, y1).predict(np.expand_dims(x_range, 1)) score_svr1 = svr.score(X1, y1) y_svr2 = svr.fit(X2, y2).predict(np.expand_dims(y_range, 1)) score_svr2 = svr.score(X2, y2) t0 = time.time() y_krb = kr.fit(Xb, yb).predict(Xpred) print("KR including bandwith fitted in %.3f s" % (time.time() - t0)) score_kr = kr.score(Xb, yb) y_kr1 = kr.fit(X1, y1).predict(np.expand_dims(x_range, 1)) score_kr1 = kr.score(X1, y1) y_kr2 = kr.fit(X2, y2).predict(np.expand_dims(y_range, 1)) score_kr2 = kr.score(X2, y2) print('R^2 / coeff determination:') print(' SVR model: cls_score=%0.3f bbox_pred=%0.3f both=%0.3f' % (score_svr1, score_svr2, score_svr)) print(' KR model: cls_score=%0.3f bbox_pred=%0.3f both=%0.3f' % (score_kr1, score_kr2, score_kr)) # R^2 / coeff determination: # SVR model: cls_score=0.675 bbox_pred=0.518 both=0.512 # KR model: cls_score=0.848 bbox_pred=0.320 both=0.881 ############################################################################### # Visualize models # fig = plt.figure() # ax = fig.gca(projection='3d') # to work in 3d # # z_surf = np.reshape(y_krb, x_surf.shape) # surf = ax.plot_surface(x_surf, y_surf, z_surf, cmap=cm.coolwarm, alpha=0.5, rstride=1, cstride=1); # plot a 3d surface plot # fig.colorbar(surf, shrink=0.5, aspect=5) # # ax.scatter(X[:,0], X[:,1], y, s=1, c='k') # plot a 3d scatter plot # # ax.set_xlabel('cls_score', fontsize=16) # ax.set_ylabel('bbox_pred', fontsize=16) # ax.set_zlabel('mAP', fontsize=16) # plt.show() fig = plt.figure() plt.scatter(X1[:, 0], y1, c='k', s=1, label='data') # plt.plot(x_range, y_kr1, c='g', label='Kernel Regression') # plt.plot(x_range, y_svr1, c='r', label='SVR') plt.xlabel('cls_score') plt.ylabel('mAP') plt.ylim(0, 0.85) # plt.title('Classification score difference as proxy for model performance/') plt.legend() plt.show() fig = plt.figure() plt.scatter(X2[:, 0], y2, c='k', s=1, label='data') # plt.plot(y_range, y_kr2, c='g', label='Kernel Regression') # plt.plot(y_range, y_svr2, c='r', label='SVR') plt.xlabel('bbox_pred') plt.ylabel('mAP') plt.ylim(0, 0.85) # plt.title('Kernel regression versus SVR') plt.legend() plt.show() # Visualize learning curves plt.figure() train_sizes, train_scores_svr, test_scores_svr = \ learning_curve(svr, X, y, train_sizes=np.linspace(0.1, 1, 10), scoring="neg_mean_squared_error", cv=10) train_sizes_abs, train_scores_kr, test_scores_kr = \ learning_curve(kr, X, y, train_sizes=np.linspace(0.1, 1, 10), scoring="neg_mean_squared_error", cv=10) plt.plot(train_sizes, test_scores_svr.mean(1), 'o-', color="r", label="SVR") plt.plot(train_sizes, test_scores_kr.mean(1), 'o-', color="g", label="Kernel Regression") plt.yscale("symlog", linthreshy=1e-7) plt.ylim(-10, -0.01) plt.xlabel("Training size") plt.ylabel("Mean Squared Error") plt.title('Learning curves') plt.legend(loc="best") plt.show()