# generate data set
Xc = np.array(dataset[:,0:dataset.shape[1]-1])
Yc = np.array(dataset[:,dataset.shape[1]-1])
X,x,Y,y = train_test_split(Xc,Yc,test_size = tt_ratio, random_state = int(time.time()*10000)%4294967295)
print('Number of training samples {}'.format(X.shape[0]))
# Define hyperparameters
gammaVal = 0.1
C_vals = np.logspace(0.001, 100, 5)
### RVR
# train rvr
rvr = RVR(gamma = gammaVal, kernel = 'rbf')
t1 = time.time()
mem_us_rvr = memory_usage((rvr.fit,(X,Y)),interval=0.1)
#rvr.fit(X,Y)
t2 = time.time()
t_train_rvr[itVal].append(t2-t1)
minMem_rvr = min(mem_us_rvr)
maxMem_rvr = max(mem_us_rvr)
# test rvr
t1= time.time()
rvr_errs[itVal].append(mean_squared_error(rvr.predict(x),y))
t2 = time.time()
t_test_rvr[itVal].append(t2-t1)
X_valid = dataset[:, 0:17]
y_valid = dataset[:, 17]
X_train_root = X_train
X_valid_root = X_valid
X_train, X_test, X_valid = standardize_data(copy.deepcopy(X_train_root),
X_test,
copy.deepcopy(X_valid_root))
X = X_train
Y = y_train.reshape(-1, 1)
# train rvr
rvm = GridSearchCV(RVR(),
param_grid={
'gamma': [0.1, 1, 10],
'coef0': [0.1, 1, 10]
})
#rvm = RVR(gamma = 1,kernel = 'rbf')
t1 = time.time()
rvm.fit(X_train, y_train)
t2 = time.time()
y_hat = rvm.predict(X_test)
#rvm_err = mean_squared_error(y_hat,y_test)
#rvs = np.sum(rvm.active_)
#print "RVM error on test set is {0}, number of relevant vectors is {1}, time {2}".format(rvm_err, rvs, t2 - t1)
rmse_predict = RMSE(y_test.reshape(-1, 1), y_hat)
import matplotlib.pyplot as plt
import time
from sklearn.metrics import mean_squared_error
# parameters
n = 5000
# generate data set
np.random.seed(0)
Xc = np.ones([n, 1])
Xc[:, 0] = np.linspace(-5, 5, n)
Yc = 10 * np.sinc(Xc[:, 0]) + np.random.normal(0, 1, n)
X, x, Y, y = train_test_split(Xc, Yc, test_size=0.5, random_state=0)
# train rvr
rvm = RVR(gamma=1, kernel='rbf')
t1 = time.time()
rvm.fit(X, Y)
t2 = time.time()
y_hat, var = rvm.predict_dist(x)
rvm_err = mean_squared_error(y_hat, y)
rvs = np.sum(rvm.active_)
print("RVM error on test set is {0}, number of relevant vectors is {1}, time {2}".format(rvm_err, rvs, t2 - t1))
# train svr
svr = GridSearchCV(SVR(kernel='rbf', gamma=1), param_grid={'C': [0.001, 0.1, 1, 10, 100]}, cv=10)
t1 = time.time()
svr.fit(X, Y)
t2 = time.time()
svm_err = mean_squared_error(svr.predict(x), y)
svs = svr.best_estimator_.support_vectors_.shape[0]
Xc = np.array(dataset[:, 0:shape[1] - 1])
Yc = np.array(dataset[:, shape[1] - 1])
X, x, Y, y = train_test_split(Xc, Yc, test_size=tt_ratio, random_state=0)
print(X.shape)
print(Y.shape)
## train svr
rvr = SVR(gamma=1, kernel='rbf')
mem_us_rvr = memory_usage((rvr.fit, (X, Y)), interval=0.1)
#print(mem_us_rvr)
t1 = time.time()
# ## train rvrpuudii dich hani vor de 9ee glaubs am bhf / zkb gseh aber bisch zwiit weg gsi zum rüefee
rvr = RVR(gamma=1, kernel='rbf')
mem_us_rvr = memory_usage((rvr.fit, (X, Y)), interval=0.1)
print(mem_us_rvr)
#
t1 = time.time()
rvr.fit(X, Y)
t2 = time.time()
# #
rvr_err = mean_squared_error(rvr.predict(x), y)
rvr_s = np.sum(rvr.active_)
print "RVR error on test set is {0}, number of relevant vectors is {1}, time {2}".format(
rvr_err, rvr_s, t2 - t1)
#
## train svr
svr = GridSearchCV(SVR(kernel='rbf', gamma=1),
def _fit_transform(self, Xtr, Ytr, n_drop = 100, regression_method = 'linear', regression_parameters = None,
embedding = 'identity', n_dim = 3, embedding_parameters = None):
n_data, dim_data = Xtr.shape
_, dim_output = Ytr.shape
self._dim_output = dim_output
# If this is the first time the network is tuned, set the input weights.
# The weights are dense and uniformly distributed in [-1.0, 1.0]
if (self._esn_list[0]._input_weights is None):
for i in range(len(self._esn_list)):
curESN = self._esn_list[i]
if i==0:
curESN._input_weights = 2.0*np.random.rand(curESN._n_internal_units, dim_data) - 1.0
else:
prevESN = self._esn_list[i-1]
curESN._input_weights = 2.0*np.random.rand(curESN._n_internal_units, prevESN._n_internal_units) - 1.0
if (len(self._esn_list)<2 or self._esn_list[1]._delay_weights is None):
for i in range(1,len(self._esn_list)):
curESN = self._esn_list[i]
curESN._delay_weights = 2.0*np.random.rand(curESN._n_internal_units, dim_data) - 1.0
# Initialize regression method
if (regression_method == 'linear'):
# Use canonical linear regression
self._regression_method = LinearRegression()
elif (regression_method == 'ridge'):
# Ridge regression
self._regression_method = Ridge(alpha=regression_parameters)
elif (regression_method == 'bayeridge'):
lambda_1, lambda_2, alpha_1, alpha_2 = regression_parameters
#self._regression_method = BayesianRidge(lambda_1=lambda_1 , lambda_2=lambda_2 , alpha_1=alpha_1 , alpha_2=alpha_2)
self._regression_method = BayesianRidge()
elif (regression_method == 'rvm'):
self._regression_method = GridSearchCV(RVR(coef0=0.01), param_grid={'degree': [1, 2, 3, 4],
'kernel': ['rbf'],
'gamma': [0.1, 1, 10]})
elif (regression_method == 'enet'):
# Elastic net
alpha, l1_ratio = regression_parameters
self._regression_method = ElasticNet(alpha = alpha, l1_ratio = l1_ratio)
elif (regression_method == 'ridge'):
# Ridge regression
self._regression_method = Ridge(alpha = regression_parameters)
elif (regression_method == 'lasso'):
# LASSO
self._regression_method = Lasso(alpha = regression_parameters)
elif (regression_method == 'nusvr'):
# NuSVR, RBF kernel
C, nu, gamma = regression_parameters
self._regression_method = NuSVR(C = C, nu = nu, gamma = gamma)
elif (regression_method == 'linsvr'):
# NuSVR, linear kernel
C = regression_parameters[0]
nu = regression_parameters[1]
self._regression_method = NuSVR(C = C, nu = nu, kernel='linear')
elif (regression_method == 'huber'):
# Huber Regressor
alpha, epsilon = regression_parameters
self._regression_method = HuberRegressor(epsilon=epsilon, alpha=alpha)
elif (regression_method == 'ard'):
# Huber Regressor
self._regression_method = ARDRegression()
elif (regression_method == 'kernelridge'):
# Huber Regressor
self._regression_method = KernelRidge()
else:
# Error: unknown regression method
print('Unknown regression method',regression_method)
# Initialize embedding method
if (embedding == 'identity'):
self._embedding_dimensions = self._esn_list[-1]._n_internal_units
else:
self._embedding_dimensions = n_dim
if (embedding == 'kpca'):
# Kernel PCA with RBF kernel
self._embedding_method = KernelPCA(n_components=n_dim, kernel='rbf', gamma=embedding_parameters)
elif (embedding == 'pca'):
# PCA
self._embedding_method = PCA(n_components=n_dim)
elif (embedding == 'spca'):
# Sparse PCA
self._embedding_method = SparsePCA(n_components=n_dim, alpha=embedding_parameters)
elif (embedding == 'tsvd'):
# Sparse PCA
self._embedding_method = TruncatedSVD(n_components = n_dim)
else:
raise (ValueError, "Unknown embedding method")
# Calculate states/embedded states.
states, embedded_states, _ = self._compute_state_matrix(X=Xtr, Y=Ytr, n_drop=n_drop)
# Train output
firstESN = self._esn_list[0]
self._regression_method.fit(np.concatenate(
(embedded_states, self._scale(Xtr[n_drop:, :], firstESN._input_scaling)), axis=1),
self._scale(Ytr[n_drop:, :], self._teacher_scaling).flatten())
return states, embedded_states
print(type(countryGDI))
print(countryGDI.shape)
countryHDI_f = np.transpose(np.array([countryHDI_f]))
print(type(countryHDI_f))
print(countryHDI_f.shape)
# Change value
x_val = countryGDI
y_val = countryHDI_f
# Define Regression Function
svr_lin = SVR(kernel='linear', C=1e3)
svr_poly = SVR(kernel='poly', C=1e3, degree=3)
svr_rbf = SVR(kernel='rbf', C=1e2, gamma=0.1)
lsrvr = RVR(kernel='rbf', gamma=0.1) ### CHANGE TO RVR
# Proceed regression using Support Vector Regression (SVR)
t1 = time.time()
y_rbf = svr_rbf.fit(x_val, y_val).predict(x_val)
t2 = time.time()
t_svr_rbf = t2 - t1
print('Support Vector Regression with RBF kernel takes {} s'.format(t_svr_rbf))
t1 = time.time()
y_lin = svr_lin.fit(x_val, y_val).predict(x_val)
t2 = time.time()
t_svr_lin = t2 - t1
print('Support Vector Regression with linear kernel takes {} s'.format(
t_svr_lin))
plt.close()
# parameters
n = 100
# generate data set
xMin, xMax = -5, 5
np.random.seed(5)
Xc = np.ones([n, 1])
Xc[:, 0] = np.linspace(xMin, xMax, n)
Yc = 10 * np.sinc(Xc[:, 0]) + np.random.normal(0, 1, n)
X, x, Y, y = train_test_split(Xc, Yc, test_size=0.5, random_state=0)
# train rvr
rvr = RVR(gamma=1, kernel='rbf')
mem_us_rvr = memory_usage((rvr.fit, (X, Y)), interval=0.1)
print(mem_us_rvr)
t1 = time.time()
rvr.fit(X, Y)
t2 = time.time()
rvr_err = mean_squared_error(rvr.predict(x), y)
rvs = np.sum(rvr.active_)
print "RVR error on test set is {0}, number of relevant vectors is {1}, time {2}".format(
rvr_err, rvs, t2 - t1)
# train svr
svr = GridSearchCV(SVR(kernel='rbf', gamma=1),
param_grid={'C': [0.001, 0.1, 1, 10, 100]},
# generate data set
Xc = np.array(dataset[:,0:dataset.shape[1]-1])
Yc = np.array(dataset[:,dataset.shape[1]-1])
X,x,Y,y = train_test_split(Xc,Yc,test_size = tt_ratio, random_state = int(time.time()*10000)%4294967295)
print('Number of training samples {}'.format(X.shape[0]))
# Define hyperparameters
gammaVal = 0.1
C_vals = np.logspace(0.001, 100, 5)
### RVR
# train rvr
rvr = GridSearchCV(RVR(kernel = 'rbf'), param_grid={'gamma',:gammaVals}, cv = 10)
t1 = time.time()
mem_us_rvr = memory_usage((rvr.fit,(X,Y)),interval=0.1)
#rvr.fit(X,Y)
t2 = time.time()
t_train_rvr[itVal].append(t2-t1)
minMem_rvr = min(mem_us_rvr)
maxMem_rvr = max(mem_us_rvr)
# test rvr
t1= time.time()
rvr_errs[itVal].append(mean_squared_error(rvr.predict(x),y))
t2 = time.time()
t_test_rvr[itVal].append(t2-t1)
# generate data set
#Xc = np.expand_dims(dataset[:,5], axis=1)
Xc = np.array(dataset[:,0:shape[1]-1])
Yc = np.array(dataset[:,shape[1]-1])
MAXINT = 4294967295
X,x,Y,y = train_test_split(Xc,Yc,test_size = tt_ratio,
random_state = int(time.time()*10000)%MAXINT)
print('Number of training samples {}'.format(X.shape[0]))
# Define hyperparameters
gammaVal = 0.1
# ## train rvr
rvr = RVR(gamma = gammaVal, kernel = 'rbf')
t1 = time.time()
mem_us_rvr = memory_usage((rvr.fit,(X,Y)),interval=0.1)
#rvr.fit(X,Y)
t2 = time.time()
minMem_rvr = min(mem_us_rvr)
maxMem_rvr = max(mem_us_rvr)
rvr_err = mean_squared_error(rvr.predict(x),y)
rvr_s = np.sum(rvr.active_)
print "RVR -- NMSR {0}, # SV {1}, time {2}, min Memroy {3}, max Memory {4}".format(rvr_err, rvr_s, t2 - t1,minMem_rvr, maxMem_rvr)
#
x2_val = [i / N_range * 2 * pi for i in range(N_range)]
print(type(x_val))
print(x_val.shape)
print(type(y_val))
print(y_val.shape)
# Change value
#x_val = countryGDI
#y_val = countryHDI_f
# Define Regression Function
svr_lin = SVR(kernel='linear', C=1e3)
svr_poly = SVR(kernel='poly', C=1e3, degree=3)
svr_rbf = SVR(kernel='rbf', C=1e3, gamma=0.1)
rvm = RVR(kernel='rbf', gamma=1) ### CHANGE TO RVR
# Proceed regression using Support Vector Regression (SVR)
t1 = time.time()
y_rbf = svr_rbf.fit(x_val, y_val).predict(x_val)
t2 = time.time()
t_svr_rbf = t2 - t1
print('Support Vector Regression with RBF kernel takes {} s'.format(t_svr_rbf))
t1 = time.time()
y_lin = svr_lin.fit(x_val, y_val).predict(x_val)
t2 = time.time()
t_svr_lin = t2 - t1
print('Support Vector Regression with linear kernel takes {} s'.format(
t_svr_lin))