def _helper_ss_sparse_compare_gradients(x_data, y_data, kernel, p_param_vector, p_step=1e-6):
"""
"""
k1 = kernel.copy()
ss_model = GPy.models.StateSpace( x_data, y_data, kernel=k1, noise_var=1.0, balance=False, kalman_filter_type = 'svd')
ss_model.param_array[:] = p_param_vector
ss_model.parameters_changed()
print(ss_model)
print(ss_model.checkgrad(verbose=True,step=p_step))
k2 = kernel.copy()
import GPy.models.ss_sparse_model as ss_sparse_model
sparse_model = ss_sparse_model.SparcePrecisionGP(x_data,y_data,k2, noise_var=1.0, balance=False, largest_cond_num=1e+12, regularization_type=2)
sparse_model.param_array[:] = p_param_vector
sparse_model.parameters_changed()
print(sparse_model)
print(sparse_model.checkgrad(verbose=True,step=p_step))
def evaluate_optimization(p_model='ss'):
"""
1) Order of RBF kernel matters a lot. If it is larger the model can be unstable.
Balancing of RBF model may also matter.
2) Also the order of kernels matter. Changing from RBF the first or the last
matter. - this is strange and undesirable of course. Actually, this does not make
much sence since parameters from optimization are assigned in a certain order and
changing this order spoil all parameter values. This was a mistake.
3) Sparse model:
'1_ex6_new_ss_arno_optimal_bfgs__qp_bound_True': Work OK. min_reg=1e+11
No other modifications helped.
'1_ex6_new_ss_arno_optimal_scg__qp_bound_True': Work OK. Variances are very small.
min_reg=1e+15. Works also with different orders of RBF and periodic.
'1_ex6_new_ss_arno_start_bfgs__qp_bound_True': Not very OK. The mean is
also distorted by regularization. This is the first time this effect is observed!
min_reg=1e+11
If we change the order of periodic kernel 6->5 then the mean distorsion effect
gets much smaller. Other changes did not give any better results. Min reg is
also not increased.
'1_ex6_new_ss_arno_start_scg__qp_bound_True': Work OK. min_reg=1e+15
However lower min_reg do not work. E.g. min_reg=1e+13 produce an error.
min_reg=1e+11 again work and result is ok, min_reg=1e+10 and
min_reg=1e+9 again does not work.
min_reg=1e+8 - works but mean and variance are distorted, probably
because regularization type 1 turns on.
Interesting case! Unclear how regularization work then!
Errors might appear in both Cholmod and Numpy.
E.g. (min_reg=1e+13, RBF order=6, Periodic order=6, all balances=False) gives Cholmod error.
(min_reg=1e+13, RBF order=5, Periodic order=5, all balances=False)
Gives Numpy error in computing Lambdas.
"""
#file_name = '1_ex6_new_ss_arno_start_bfgs__qp_bound_True'
#file_name = '1_ex6_new_ss_arno_start_scg__qp_bound_True'
#file_name = '1_ex6_new_sparse_arno_start_scg__qp_bound_True'
file_name = '1_ex6_new_sparse_0_scg__qp_bound_True_reg1e14'
file_path = '/home/alex/Programming/python/Sparse GP/Experiemnts/Results'
results_dict = io.loadmat( os.path.join(file_path, file_name) )
# Data ->
(x_data, y_data) = load_data_mine(detrend = True,detrend_model = 'gp')
# Data <-
# predict data->
years_to_predict = 8
step = np.mean( np.diff(x_data[:,0]) )
x_new = x_data[-1,0] + np.arange( step, years_to_predict, step ); x_new.shape = (x_new.shape[0],1)
x_new = np.vstack( (x_data, x_new)) # combine train and test data
#ssm_mean, ssm_var = model.predict(x_new, include_likelihood=False, largest_cond_num=1e+15, regularization_type=2)
# predict data<-
#import pdb; pdb.set_trace()
# Kernel ->
var_1_per = 5.0
ls_1_per = 1 / 2 # division by 2 is done because meanings in parameters are different
period = 1
kernel1 = GPy.kern.sde_StdPeriodic(1, variance=var_1_per, lengthscale=ls_1_per, period=period, balance=False, approx_order = 6)
var_1_per = 1.0 # does not change
ls_1_per = 140.0
kernel2 = GPy.kern.sde_Matern32(1, variance=var_1_per, lengthscale=ls_1_per)
# Short term fluctuations kernel
var_1_per = 0.5
ls_1_per = 1.0
kernel3 = GPy.kern.sde_Matern32(1, variance=var_1_per, lengthscale=ls_1_per)
var_1_per = 1
ls_1_per = 100
kernel4 = GPy.kern.sde_RBF(1, variance=var_1_per, lengthscale=ls_1_per, balance=False, approx_order = 6)
noise_var = 1.0
kernel = kernel1 * kernel2 + kernel3# + kernel4
# Kernel <-
if p_model == 'ss':
model1 = GPy.models.StateSpace( x_data, y_data, kernel=kernel, noise_var=noise_var, balance=False, kalman_filter_type = 'svd')
model2 = GPy.models.StateSpace( x_data, y_data, kernel=kernel.copy(), noise_var=noise_var, balance=False, kalman_filter_type = 'svd')
elif p_model == 'sparse':
# Regularization assume regularization type 2
import GPy.models.ss_sparse_model as ss_sparse_model
model1 = ss_sparse_model.SparcePrecisionGP(x_data,y_data,kernel, noise_var=noise_var, balance=False, largest_cond_num=1e+14, regularization_type=2) # Arno optimal parameters
model2 = ss_sparse_model.SparcePrecisionGP(x_data,y_data,kernel.copy(), noise_var=noise_var, balance=False, largest_cond_num=1e+14, regularization_type=2) # Parameters from optimization
else:
raise ValueError("Wrong Parameter 1")
# Model <-
# Arno optimal parameters ->
model_params = io.loadmat('/home/alex/Programming/python/Sparse GP/solin-2014-supplement/arno_opt_params_to_python.mat')
tt = np.array( [ model_params['per_magnSigma2'], model_params['per_period'], model_params['per_lengthscale']/2,
1.0, model_params['quasi_per_mat32_lengthscale'],
model_params['mat32_inacc_magnSigma2'], model_params['mat32_inacc_lengthScale']*10,
#model_params['rbf_magnSigma2'], model_params['rbf_lengthscale'],
model_params['opt_noise'] ] )
#import pdb; pdb.set_trace()
#print(tt)
model1.param_array[:] = tt
model1.parameters_changed()
print(model1)
#print(model1.objective_function())
mean1, var1 = model1.predict(x_new, include_likelihood=False) #, largest_cond_num=1e+15, regularization_type=2)
#_helper_ss_sparse_compare_gradients(x_data, y_data, kernel, tt, p_step=1e-4)
# Arno optimal parameters <-
# Optmim optimal (parameters from optimization) ->
params = results_dict['params'][0]
#model = GPy.models.StateSpace( x_data, y_data, kernel=kernel, noise_var=1.0, balance=False, kalman_filter_type = 'svd')
model2.randomize()
#import pdb; pdb.set_trace()
params[6] = params[6]*10
#params = np.hstack((params[0:5],params[7]))
model2.param_array[:] = params
model2.parameters_changed()
print(model2)
mean2, var2 = model2.predict(x_new, include_likelihood=False) #, largest_cond_num=1e+15, regularization_type=2)
#_helper_ss_sparse_compare_gradients(x_data, y_data, kernel, params, p_step=1e-4)
# Optmim optimal (parameters from optimization) <-
# Plot ->
plt.figure(2)
#plt.title('Electricity Consumption Data', fontsize=30)
plt.subplot(1,2,1)
plt.plot( x_data, y_data, 'g-', label='Data',linewidth=1, markersize=5)
plt.plot( x_new, mean1, 'b-', label='Data',linewidth=1, markersize=5)
plt.plot( x_new, mean1+np.sqrt(var1), 'r--', label='Data',linewidth=1, markersize=5)
plt.plot( x_new, mean1-np.sqrt(var1), 'r--', label='Data',linewidth=1, markersize=5)
plt.title('Arno Optimal parameters')
plt.subplot(1,2,2)
plt.plot( x_data, y_data, 'g-', label='Data',linewidth=1, markersize=5)
plt.plot( x_new, mean2, 'b-', label='Data',linewidth=1, markersize=5)
plt.plot( x_new, mean2+np.sqrt(var2), 'r--', label='Data',linewidth=1, markersize=5)
plt.plot( x_new, mean2-np.sqrt(var2), 'r--', label='Data',linewidth=1, markersize=5)
plt.title('Optimization parameters')
#plt.xlabel('Time (Hours)', fontsize=25)
#plt.ylabel('Normalized Value', fontsize=25)
#plt.legend(loc=2)
plt.show()
def experiment2_optimize_hyper_parameters(hyperparameters=1, p_model='sparse', optim = 'scg',
bound_qp_var=False, detrend_model = 'gp', detrend_file=None, save_file_name_prefix=None):
"""
This function optimizes the hyperparameters of the model by gradient optimization.
Note, that the trend is fitted separately and removed from the data
before running optimization.
Starting parameters are assigned a reasonable values.
Input:
------------------------
hyperparameters: None or int
For optimization None must be fed. Number might be used if we want to
load hyperparameters from a file and then start optimzation from them.
Not used currently.
p_model: string
Defines a models, one of: 'gp', 'ss', 'sparse'
optim: string
Which optimization method is used, one of: 'scg', 'bfgs.'
bound_qp_var: bool.
Whether to bound quasi-periodic lengthscale. If not bounded some kernel
may perform badly. Usually set to true.
detrend_model: 'gp' or 'ss'
Which model is used for detrending
detrend_file: string
Which parameters to take for detrending. Note that for detrending a GP
model is used because for state-space and sparse the spamping is too
dence wrt trend lengthscale, so they might perform badly.
save_file_name_prefix: string
Prefix of the resulting file name. The full file name is printed in the end
and some other info is added to it.
Output:
--------------------------------
Optimal hyperparameters are saved to a file.
"""
#import pdb; pdb.set_trace()
np.random.seed(237)
import GPy.models.ss_sparse_model as ss_sparse_model
# Data ->
(x_data, y_data) = load_data(detrend = True, detrend_model = detrend_model, detrend_file=detrend_file)
# Data <-
#import pdb; pdb.set_trace()
# Arno start hyper parameters ->
if hyperparameters is not None:
model_params = io.loadmat('/home/alex/Programming/python/Sparse GP/solin-2014-supplement/arno_opt_params_to_python.mat')
# tt = np.array( [ model_params['per_magnSigma2'], model_params['per_period'],
# model_params['per_lengthscale']/2,
# 1.0, model_params['quasi_per_mat32_lengthscale'],
# model_params['mat32_inacc_magnSigma2'],
# model_params['mat32_inacc_lengthScale'],
# model_params['rbf_magnSigma2'], model_params['rbf_lengthscale'],
# model_params['opt_noise'] ] )
# Periodic kernel
var_1_per = 0.5 #1.0 # 5.0
ls_1_per = 1 / 2 # division by 2 is done because meanings in parameters are different
period = 1
kernel1 = GPy.kern.sde_StdPeriodic(1, variance=var_1_per, lengthscale=ls_1_per, period=period, balance=False, approx_order = 6)
if hyperparameters is not None:
kernel1.param_array[:] = np.squeeze( np.array( [ model_params['per_magnSigma2'], model_params['per_period'], model_params['per_lengthscale']/2 ] ) )
# division by 2 is only for Arno start, otherwise use just pure param.
kernel1.period.fix()
if bound_qp_var:
kernel1.lengthscale.constrain_bounded(0.4, 20000)
# Quasiperiodic part
var_1_per = 1.0 # does not change, fixed!
ls_1_per = 50.0 #140.0
kernel2 = GPy.kern.sde_Matern32(1, variance=var_1_per, lengthscale=ls_1_per)
if hyperparameters is not None:
kernel2.param_array[:] = np.array( [ 1.0, model_params['quasi_per_mat32_lengthscale'] ] )
kernel2.variance.fix()
# Short term fluctuations kernel
var_1_per = 0.1 #0.5
ls_1_per = 50
kernel3 = GPy.kern.sde_Matern32(1, variance=var_1_per, lengthscale=ls_1_per)
if hyperparameters is not None:
kernel3.param_array[:] = np.array( [ model_params['mat32_inacc_magnSigma2'],
model_params['mat32_inacc_lengthScale'] ] )
# RBF kernel. Not used!
var_1_per = 1
ls_1_per = 100
kernel4 = GPy.kern.sde_RBF(1, variance=var_1_per, lengthscale=ls_1_per, balance=False, approx_order = 6)
if hyperparameters is not None:
kernel4.param_array[:] = np.array( [ model_params['rbf_magnSigma2'],
model_params['rbf_lengthscale'] ] )
noise_var = 0.1
if hyperparameters is not None:
noise_var = float(model_params['opt_noise'])
kernel = kernel1 * kernel2 + kernel3# + kernel4
# Arno start hyper parameters <-
# Model ->
if p_model == 'ss':
model = GPy.models.StateSpace( x_data, y_data, kernel=kernel, noise_var=noise_var, balance=False, kalman_filter_type = 'svd')
elif p_model == 'sparse':
# Regularization assume regularization type 2
model = ss_sparse_model.SparcePrecisionGP(x_data,y_data,kernel, noise_var=noise_var, balance=False,
largest_cond_num=1e+14, regularization_type=2)
else:
raise ValueError("Wrong Parameter 1")
# Model <-
# Model ->
#import pdb; pdb.set_trace()
# model.kern.mul.Mat32.variance.fix()
# model.kern.mul.std_periodic.period.fix()
print(model)
# Model <-
class observer(object):
def __init__(self):
"""
"""
self.opt_obj_grads = None
self.opt_gradients = None
self.opt_params = None
self.opt_obj_funcs = None
def notif_callable(self, me, which=None):
"""
Description
"""
#import pdb; pdb.set_trace()
# me.obj_grads
if isinstance(self.opt_obj_grads , np.ndarray): # previous array
self.opt_obj_grads = np.vstack( (self.opt_obj_grads, me.obj_grads) )
else:
rr = me.obj_grads
if isinstance(self.opt_obj_grads , list):
if isinstance(rr, np.ndarray):
tt = np.empty( (len(self.opt_obj_grads), rr.shape[0]) ) * np.nan
self.opt_obj_grads = np.vstack( (tt, rr) )
else:
self.opt_obj_grads.append( (rr if rr is not None else np.nan) )
else:
if isinstance(rr , np.ndarray):
self.opt_obj_grads = rr
else:
self.opt_obj_grads = [ (rr if (rr is not None) else np.nan), ]
# me.gradient
if isinstance(self.opt_gradients , np.ndarray): # previous array
self.opt_gradients = np.vstack( (self.opt_gradients, me.gradient) )
else:
rr = me.gradient # same as me.objective_function_gradients()
if isinstance(self.opt_gradients , list):
if isinstance(rr, np.ndarray):
tt = np.empty( (len(self.opt_gradients), rr.shape[0]) ) * np.nan
self.opt_gradients = np.vstack( (tt, rr) )
else:
self.opt_gradients.append( (rr if rr is not None else np.nan) )
else:
if isinstance(rr , np.ndarray):
self.opt_gradients = rr
else:
self.opt_gradients = [ (rr if (rr is not None) else np.nan), ]
# me.param_array
if isinstance(self.opt_params , np.ndarray): # previous array
self.opt_params = np.vstack( (self.opt_params, me.param_array) )
else:
rr = me.param_array # same as me.objective_function_gradients()
if isinstance(self.opt_params , list):
if isinstance(rr, np.ndarray):
tt = np.empty( (len(self.opt_params), rr.shape[0]) ) * np.nan
self.opt_params = np.vstack( (tt, rr) )
else:
self.opt_params.append( (rr if rr is not None else np.nan) )
else:
if isinstance(rr , np.ndarray):
self.opt_params = rr
else:
self.opt_params = [ (rr if (rr is not None) else np.nan), ]
if self.opt_obj_funcs is None: # first iteration
self.opt_obj_funcs = [me.objective_function(),] if (me.objective_function() is not None) else [np.nan,]
else:
self.opt_obj_funcs.append( me.objective_function() )
def save(self, optim_save_path, optim_file_name, save= False):
"""
Saves optim data to file
"""
result_dict = {}
result_dict['opt_obj_grads'] = self.opt_obj_grads
result_dict['opt_gradients'] = self.opt_gradients
result_dict['opt_params'] = self.opt_params
result_dict['opt_obj_funcs'] = np.array(self.opt_obj_funcs)
optim_file_name = optim_file_name + '__optim_history'
if save:
io.savemat(os.path.join(optim_save_path, optim_file_name), result_dict)
return result_dict
oo = observer()
model.add_observer( oo, oo.notif_callable )
if optim == 'bfgs':
# L-BFGS-B optimization ->
#' factr' - criteria for stoping. If eps * factr < (f_k - f_(k+1)) then stop
# default: 1e7
#
# pgtol : float, optional
# The iteration will stop when
# ``max{|proj g_i | i = 1, ..., n} <= pgtol``
# where ``pg_i`` is the i-th component of the projected gradient.
# default: 1e-5
model.optimize(optimizer ='lbfgsb', bfgs_factor = 1e12, gtol=1e-3, messages=True)
# L-BFGS-B optimization <-
elif optim == 'scg':
# SCG optimization ->
# xtol: controls reduction in step size
# default: 1e-6
# ftol: controls reduction in objective
# default: 1e-6
# gtol: controls reduction in gradients
# default: 1e-5
model.optimize(optimizer ='scg', xtol = 1e-3, ftol=1e-3, gtol = 1e-3, messages=True)
# SCG optimization <-
else:
raise ValueError("Wrong Optimizer name")
#model.optimize(optimizer ='scg', messages=True)
print(model)
result_dict = {}
result_dict['params'] = model.param_array[:]
result_dict.update( oo.save('', '', save= False) )
hyp = str(hyperparameters) if hyperparameters is not None else '2'
save_file_name = save_file_name_prefix + '_' +p_model+ '_' + hyp + '_' + optim + '_' + '_qp_bound_' +str(bound_qp_var)
print(results_files_prefix)
print(save_file_name)
io.savemat(os.path.join(results_files_prefix, save_file_name), result_dict)
return model, oo
def experiment2_fit_sparse_gp(hyperparameters='sparse_optimal',detrend_file_name=None, other_params_file_name=None):
"""
The model is Sparse GP.
Fit the model to the data and plot the result.
Input:
-------------------------------------------
hyperparameters: string
Which parameters (hyper-parameters) to use for fitting.
Two options: 'sparse_optimal' and 'arno_optimal'. The first one is used here.
The file name of the parameters is written in the text.
The second one is older option for comparison.
detrend_file_name: string
Detrended file name.
other_params_file_nameL string
Other params file name.
Output:
--------------------------------------------
Fitting plot
"""
import time
import GPy.models.ss_sparse_model as ss_sparse_model
# Data ->
(x_data, y_data) = load_data(detrend = False)
# Data <-
if hyperparameters == 'sparse_optimal':
#detrend_file_name = 'ex6_trend_fit_sparse_2.mat'
#other_params_file_name = '1_ex6_new_sparse_0_scg__qp_bound_True_reg1e14'
#other_params_file_name = '1_ex6_new_ss_2_scg__qp_bound_True'
#other_params_file_name = '1_ex6_new_sparse_2_scg__qp_bound_True'
trend_params = io.loadmat( os.path.join(results_files_prefix, detrend_file_name) );
trend_params = trend_params['params'][0]
other_params = io.loadmat( os.path.join(results_files_prefix, other_params_file_name) )
other_params = other_params['params'][0]
var_1_per = 1
ls_1_per = 1 / 2 # division by 2 is done because meanings in parameters are different
period = 1
kernel1 = GPy.kern.sde_StdPeriodic(1, variance=var_1_per, lengthscale=ls_1_per, period=period, balance=False, approx_order = 6)
var_1_quasi = 1 # does not change
ls_1_quasi = 1
kernel2 = GPy.kern.sde_Matern32(1, variance=var_1_quasi, lengthscale=ls_1_quasi)
# Short term fluctuations kernel
var_1_short = 1
ls_1_short = 1
kernel3 = GPy.kern.sde_Matern32(1, variance=var_1_short, lengthscale=ls_1_short)
# RBF kernel. Not used if the parameter without_rbf = True
var_1_trend = trend_params[0]
ls_1_trend = trend_params[1]
kernel4 = GPy.kern.sde_RBF(1, variance=var_1_trend, lengthscale=ls_1_trend, balance=False,
approx_order = 6)
kernel = kernel1 * kernel2 + kernel3
kernel.param_array[:] = other_params[:-1]
kernel = kernel + kernel4
kernel.fix() # fix all the parameters except the noise. Relevant when optimizing noise below.
noise_var = other_params[-1]
model = ss_sparse_model.SparcePrecisionGP(x_data,y_data,kernel, noise_var=noise_var, balance=False, largest_cond_num=1e+14, regularization_type=2)
#model.optimize(messages=True)
# The value below is obtained my uncommenting the line above and performing optimization.
model.Gaussian_noise = 0.000358573407244 # recompute
print('Run sparse model')
elif hyperparameters == 'arno_optimal':
model_params = io.loadmat('/home/alex/Programming/python/Sparse GP/solin-2014-supplement/arno_opt_params_to_python.mat')
var_1_per = 1
ls_1_per = 1 / 2 # division by 2 is done because meanings in parameters are different
period = 1
kernel1 = GPy.kern.sde_StdPeriodic(1, variance=var_1_per, lengthscale=ls_1_per, period=period, balance=False, approx_order = 6)
var_1_per = 1 # does not change
ls_1_per = 1
kernel2 = GPy.kern.sde_Matern32(1, variance=var_1_per, lengthscale=ls_1_per)
# Short term fluctuations kernel
var_1_per = 1
ls_1_per = 1
kernel3 = GPy.kern.sde_Matern32(1, variance=var_1_per, lengthscale=ls_1_per)
# RBF kernel. Not used if the parameter without_rbf = True
var_1_per = 1
ls_1_per = 1
kernel4 = GPy.kern.sde_RBF(1, variance=var_1_per, lengthscale=ls_1_per, balance=False, approx_order = 6)
# RBF kernel. Not used if the parameter
kernel = kernel1 * kernel2 + kernel3 + kernel4
#kernel = kernel1 * kernel2 + kernel4
model = ss_sparse_model.SparcePrecisionGP(x_data,y_data,kernel, noise_var=1.0, balance=False, largest_cond_num=1e+17, regularization_type=2)
#import pdb; pdb.set_trace()
model.param_array[:] = np.array( [ model_params['per_magnSigma2'], model_params['per_period'], model_params['per_lengthscale']/2,
1.0, model_params['quasi_per_mat32_lengthscale'],
model_params['mat32_inacc_magnSigma2'], model_params['mat32_inacc_lengthScale']*50,
model_params['rbf_magnSigma2'], model_params['rbf_lengthscale'],
model_params['opt_noise'] ] )
print(model)
# predict ->
years_to_predict = 8
step = np.mean( np.diff(x_data[:,0]) )
x_new = x_data[-1,0] + np.arange( step, years_to_predict, step ); x_new.shape = (x_new.shape[0],1)
x_new = np.vstack( (x_data, x_new)) # combine train and test data
#import pdb; pdb.set_trace()
t1 = time.time()
sparse_mean, sparse_var = model.predict(x_new, include_likelihood=True, balance=False,
largest_cond_num=1e+14, regularization_type=2)
print('Sparse GP prediction time {0:e} sec.'.format(time.time() - t1))
# predict <-
# # Plot ->
# sparse_mean,sparse_std,x_data_denorm,y_data_denorm = denormalize_data(sparse_mean,np.sqrt(sparse_var))
# x_new = x_new + 1974 # Put the right year
#
# plt.figure(2)
# #plt.title('Electricity Consumption Data', fontsize=30)
# plt.plot( x_data_denorm, y_data_denorm, 'g-', label='Data',linewidth=1, markersize=5)
# plt.plot( x_new, sparse_mean, 'b-', label='Mean',linewidth=1, markersize=5)
# plt.plot( x_new, sparse_mean+2*sparse_std, 'r--', label='2*Std',linewidth=1, markersize=5)
# plt.plot( x_new, sparse_mean-2*sparse_std, 'r--', label='2*Std',linewidth=1, markersize=5)
# plt.xlim((2010,2026))
# plt.ylim((370,450))
# #plt.xlabel('Time (Hours)', fontsize=25)
# #plt.ylabel('Normalized Value', fontsize=25)
# #plt.legend(loc=2)
# plt.show()
# # Plot <-
# Plot ->
sparse_mean,sparse_std,x_data_denorm,y_data_denorm = denormalize_data(sparse_mean,np.sqrt(sparse_var))
x_new = x_new + 1974 # Put the right year
plt.figure(2)
(y_min,y_max) = (370,450)
(x_min,x_max) = (2010,2026)
title_font = {'family': 'serif', 'color': 'k','weight': 'normal', 'size': 22}
plt.title(r'Spin-GP prediction', fontdict=title_font)
plt.plot( x_data_denorm, y_data_denorm, label='Data', marker= '*', color='0', markersize=5, linewidth=0)
plt.plot( x_new, sparse_mean, color=(0,0,1), linestyle='-', label=r'$m$ (prediction mean)',linewidth=1, markersize=5)
plt.plot( x_new, sparse_mean+2*sparse_std, color=(0,0,0.5), linestyle='--', label=r'$m\pm2\sigma$',linewidth=1)
plt.plot( x_new, sparse_mean-2*sparse_std, color=(0,0,0.5), linestyle='--', label=None,linewidth=1)
plt.plot( (x_data_denorm[-1], x_data_denorm[-1]),(380,440), color=(0.2,0.2,0.2), linestyle='--', linewidth=1,
label='Data/Prediction delimiter')
plt.xlim( (x_min,x_max) )
plt.ylim( (y_min,y_max) )
labels_font = {'family': 'serif', 'color': 'k','weight': 'normal', 'size': 20}
plt.xlabel(r'Time (year)', fontdict=labels_font)
plt.ylabel(r'$\mathrm{CO_2}$ concentration (ppm)', fontdict=labels_font)
plt.tight_layout() # for adjusting the bottom of x label
# legend ->
plt.legend(loc=4)
# legend <-
# Grid ->
plt.grid(False)
# Grid <-
# Ticks ->
ax = plt.gca()
from matplotlib.ticker import MultipleLocator, AutoMinorLocator
x_major = MultipleLocator(4); x_minor = MultipleLocator(2)
ax.xaxis.set_major_locator(x_major); ax.xaxis.set_minor_locator(x_minor)
y_major = MultipleLocator(20); y_minor = MultipleLocator(10)
ax.yaxis.set_major_locator(y_major); ax.yaxis.set_minor_locator(y_minor)
#ax.xaxis.set_minor_locator(AutoMinorLocator(2))
plt.tick_params(which='both', width=1)
plt.tick_params(which='major', length=7)
plt.tick_params(which='minor', length=4)
plt.yticks( range(370,470,20), (str(ss) for ss in range(370,470,20)), fontsize=20)
plt.xticks([2010,2014,2018,2022,2026], ['2010','2014','2018','2022','2026'],fontsize=20)
# Ticks <-
plt.show()
def load_data(detrend = False, detrend_model = 'gp', detrend_file='ex6_trend_fit_sparse_2.mat',plot=False):
"""
The function loads the raw data, normalize it (zero mean, unit variance)
and optionally detrends it.
Input:
-------------------------
detrend: bool
Whether to detrend or not
detrend_model: which model is used for detrending. Sparse model and StateSpace model
may be not good for detrending because data dt is too small wrt the trend lengthscale.
detrend_model: string
Which model is used for detrending.
detrend_file: string
file_name of the trend parameters. Parameters are found in the function experiment2_fit_trend.
See also the variable
plot: bool
Whether to plot the trend.
"""
#import pdb; pdb.set_trace()
x_data, y_data = load_data_low_level(data_file_path)
y_data = (y_data - np.mean(y_data)) / np.std(y_data)
x_data = x_data - 1974 # Start time from 0.
#import pdb; pdb.set_trace()
if detrend:
np.random.seed(234) # seed the random number generator
results_dict = io.loadmat( os.path.join(results_files_prefix, detrend_file) )
params = results_dict['params'][0]
variance_init = float(params[0])
lengthcale_init = float(params[1])
noise_var_init = float(params[2])
if detrend_model == 'gp':
kernel = GPy.kern.RBF(1,variance=variance_init, lengthscale = lengthcale_init)
kernel.variance.fix()
kernel.lengthscale.fix()
model = GPy.models.GPRegression(x_data, y_data, kernel, noise_var=noise_var_init)
model.optimize()
elif detrend_model == 'ss':
kernel = GPy.kern.sde_RBF(1,variance=variance_init, lengthscale = lengthcale_init,
balance= False, approx_order = 6 )
kernel.variance.fix()
kernel.lengthscale.fix()
model = GPy.models.StateSpace(x_data, y_data, kernel, noise_var=noise_var_init, balance=False, kalman_filter_type = 'svd')
elif detrend_model == 'sparse':
import GPy.models.ss_sparse_model as ss_sparse_model
kernel = GPy.kern.sde_RBF(1,variance=variance_init, lengthscale = lengthcale_init,
balance= False, approx_order = 6 )
kernel.variance.fix()
kernel.lengthscale.fix()
model = ss_sparse_model.SparcePrecisionGP(x_data,y_data,kernel, noise_var=noise_var_init,
balance=False,
largest_cond_num=1e+20, regularization_type=2)
#import pdb; pdb.set_trace()
print('Detrend:')
print(model)
(y_pred,var_pred) = model.predict(x_data)
if plot:
plt.figure(1)
plt.plot( x_data, y_data, 'b.-', label='Data',linewidth=1, markersize=5)
plt.plot( x_data, y_pred, 'r-', label='Data',linewidth=1, markersize=5)
plt.plot( x_data, y_pred+np.sqrt(var_pred), 'r--', label='Data',linewidth=1, markersize=5)
plt.plot( x_data, y_pred-np.sqrt(var_pred), 'r--', label='Data',linewidth=1, markersize=5)
plt.show()
y_data = y_data - y_pred
return x_data, y_data
def experiment2_fit_trend(trend_model='gp', optimize=False, optimize_file_prefix = '1', load_params=True, load_params_file_name=None):
"""
This function fits the trend to the original raw data.
This function must be run in order to find the parameters (hyperparameters)
of the trend.
Input:
------------------
trend_model: string
Which model is used for fitting
optimize: bool
Whether to perform trend parameters optimization.
optimize_file_prefix: string
When parameters optimized this is the prefix of the file with optimal parameters.
Note, that 'trend_model' is added to the end or the file name anyway.
load_params:
Whether the trend parameters are loaded from the file
load_params_file_name: string
If parameters are loaded from the file, this tells the file name of the parameters.
Output:
----------------------------
The output is the plot of the trend(if no optimization happen) or
optimization is done and parameters are saved in the file. File name see in the
code.
"""
(x_data,y_data) = load_data(detrend = False) # loads the raw data
np.random.seed(234) # seed the random number generator just in case there are some initializations.
if not load_params:
variance_init = 40
lengthcale_init = 400
noise_var_init = 0.1
else:
#import pdb; pdb; pdb.set_trace()
results_dict = io.loadmat( os.path.join(results_files_prefix, load_params_file_name) )
params = results_dict['params'][0]
variance_init = float(params[0])
lengthcale_init = float(params[1])
noise_var_init = float(params[2])
if trend_model == 'gp':
kernel = GPy.kern.RBF(1,variance=variance_init, lengthscale = lengthcale_init)
model = GPy.models.GPRegression(x_data, y_data, kernel, noise_var=noise_var_init)
elif trend_model == 'ss':
if optimize:
x_data = x_data[::10]; y_data = y_data[::10]
kernel = GPy.kern.sde_RBF(1,variance=variance_init, lengthscale = lengthcale_init,
balance= False, approx_order = 6 )
kernel.lengthscale.constrain_bounded(100,500)
model = GPy.models.StateSpace(x_data, y_data, kernel, noise_var=noise_var_init, balance=False, kalman_filter_type = 'svd')
elif trend_model == 'sparse':
if optimize:
x_data = x_data[::40]; y_data = y_data[::40]
import GPy.models.ss_sparse_model as ss_sparse_model
kernel = GPy.kern.sde_RBF(1,variance=variance_init, lengthscale = lengthcale_init,
balance= False, approx_order = 6 )
kernel.lengthscale.constrain_bounded(100,500)
model = ss_sparse_model.SparcePrecisionGP(x_data,y_data,kernel, noise_var=noise_var_init,
balance=False,
largest_cond_num=1e+20, regularization_type=2)
#import pdb; pdb.set_trace()
if optimize:
model.optimize(messages=True)
print(model)
#import pdb; pdb.set_trace()
result_dict = {}
result_dict['params'] = model.param_array[:]
save_file_name = optimize_file_prefix + '_' + trend_model
print(save_file_name)
io.savemat(os.path.join(results_files_prefix, save_file_name), result_dict)
else:
model.parameters_changed()
print(model)
if trend_model == 'gp':
(y_pred,var_pred) = model.predict(x_data)
elif trend_model == 'ss':
(y_pred,var_pred) = model.predict(x_data, include_likelihood=True)
elif trend_model == 'sparse':
(y_pred,var_pred) = model.predict(x_data, include_likelihood=True, balance=False,
largest_cond_num=1e+14, regularization_type=2)
#(y_pred,var_pred) = model.predict(x_data)
plt.figure(1)
plt.plot( x_data, y_data, 'b.-', label='Data',linewidth=1, markersize=5)
plt.plot( x_data, y_pred, 'r-', label='Data',linewidth=1, markersize=5)
plt.plot( x_data, y_pred+np.sqrt(var_pred), 'r--', label='Data',linewidth=1, markersize=5)
plt.plot( x_data, y_pred-np.sqrt(var_pred), 'r--', label='Data',linewidth=1, markersize=5)
plt.show()
def model_time_measurement(n_points,
kernel_num=0,
repetitions_num=3,
run_sparse=True,
run_ss=True,
run_gp=True):
"""
This function builds a model (in GPy sense) and measure the time it took.
Input:
------------------------------
n_points: int
Number of data points in the data
kenrel_num: int
Which kernel to use. For State-space and sparse model the kernel defines
the block size in the inverse kernel BTD matrix.
repetitions_num: int
How many times time measurement is performed
run_sparse: bool
Whether to run the time measurement for SpiIn-GP
run_ss: bool
Whether to run the time measurement for State-space model
run_gp: bool
Whether to run the time measurement for State-space model
Output:
----------------------
Mean running time
"""
print(
'Time measurement test: %i data points, run_sparse %i, run_ss %i , run_gp %i'
% (n_points, run_sparse, run_ss, run_gp))
x_data, y_data = generate_data(n_points,
p_x_lower_value=0.0,
p_x_upper_value=20000.0)
kernel1, kernel2 = select_kernel(kernel_num)
noise_var = 0.1
sparse_run_time = []
ss_run_time = []
gp_run_time = []
gc.collect()
for rep_no in range(repetitions_num):
if run_sparse:
#print('Sparse GP run:')
kern = kernel1.copy()
t1 = time.time()
sparse_gp = ss_sparse_model.SparcePrecisionGP(
x_data,
y_data,
kernel=kern,
noise_var=noise_var,
balance=False,
largest_cond_num=1e+10,
regularization_type=2)
sparse_run_time.append(time.time() - t1)
sparse_gp_marginal_ll = (-sparse_gp.objective_function())
sparse_d_marginal_ll = -sparse_gp.objective_function_gradients()
sparse_d_marginal_ll.shape = (sparse_d_marginal_ll.shape[0], 1)
del sparse_gp, kern
gc.collect()
if run_ss:
#print('SS run:')
kern = kernel1.copy()
t1 = time.time()
ss_model = GPy.models.StateSpace(x_data,
y_data,
kernel=kern,
noise_var=noise_var,
balance=False,
kalman_filter_type='svd')
ss_run_time.append(time.time() - t1)
#ss_marginal_ll = (-ss_model.objective_function())
#ss_d_marginal_ll = -ss_model.objective_function_gradients(); ss_d_marginal_ll.shape = (ss_d_marginal_ll.shape[0],1)
del ss_model, kern
gc.collect()
if run_gp:
#print('Regular GP run:')
kern = kernel2.copy()
t1 = time.time()
gp_reg = GPy.models.GPRegression(x_data,
y_data,
kern,
noise_var=noise_var)
gp_run_time.append(time.time() - t1)
#gp_marginal_ll = (-gp_reg.objective_function())
#gp_d_marginal_ll = -gp_reg.objective_function_gradients(); gp_d_marginal_ll.shape = (gp_d_marginal_ll.shape[0],1)
del gp_reg, kern
gc.collect()
gc.collect()
return sparse_run_time, ss_run_time, gp_run_time
def experiment6_action1(hyperparameters='arno_start', p_model='ss', optim = 'bfgs', bound_qp_var=False):
"""
Copies the first submision experiment6 action 1. Training of hyper parameters.
It was working improperly
earlier. What is now? - Working.
"""
import GPy.models.ss_sparse_model as ss_sparse_model
# Data ->
data_file_path= '/home/agrigori/Programming/python/Sparse GP/CO2_data/co2_weekly_init_clean.csv'
results_filex_prefix = '/home/agrigori/Programming/python/Sparse GP/Experiemnts/Results'
data = np.loadtxt(data_file_path);
data = data[ np.where(data[:,1] > 0)[0] ,:] # get rid of missing values
y_data = data[:,1]; y_data.shape = (y_data.shape[0],1)
x_data = data[:,0]; x_data.shape = (x_data.shape[0],1)
y_data = (y_data - np.mean(y_data)) / np.std(y_data)
x_data = x_data - 1974
# Data <-
if hyperparameters == 'arno_start':
# Start hyper parameters which were used in the
var_1_trend = 1e4
ls_1_trend = 100.0
per_per = 1 # fixed
per_var = 5.0
per_ls = 1
var_2_quasi = 1.0
ls_2_quasi = 140
var_3_quasi = 0.5
ls_3_quasi = 1
noise_var = 1.0
kernel = GPy.kern.sde_StdPeriodic(1, variance=per_var, period = per_per, lengthscale=per_ls, balance=False, approx_order = 6)*\
GPy.kern.sde_Matern32(1, variance=var_2_quasi, lengthscale=ls_2_quasi) +\
GPy.kern.sde_Matern32(1, variance=var_3_quasi, lengthscale=ls_3_quasi) +\
GPy.kern.sde_RBF(1, variance=var_1_trend, lengthscale=ls_1_trend, balance=False, approx_order = 6)
kernel.mul.Mat32.variance.fix()
kernel.mul.std_periodic.period.fix()
if bound_qp_var:
kernel.mul.std_periodic.lengthscale.constrain_bounded(0.2, 20000)
# result_dict = io.loadmat(os.path.join(results_filex_prefix,'ex6_params'))
# params = result_dict['params'].squeeze()
#
# kernel[:] = params[0:-1]
# noise_var = params[-1]
if p_model == 'ss':
model = GPy.models.StateSpace( x_data, y_data, kernel=kernel, noise_var=noise_var, balance=False, kalman_filter_type = 'svd')
elif p_model == 'sparse':
# Regularization assume regularization type 2
model = ss_sparse_model.SparcePrecisionGP(x_data,y_data,kernel, noise_var=1.0, balance=False, largest_cond_num=1e+11, regularization_type=2)
# optim scg: Kernel: std_per approx order=6 bal=False, rbf_approx_order=6, bal=False. Model: balance=False, largest_cond_num=1e+12, regularization_type=2.
# optim bfgs: Kernel: std_per approx order=6 bal=False, rbf_approx_order=5, bal=False. Model: balance=False, largest_cond_num=1e+11, regularization_type=2.
else:
raise ValueError("Wrong Parameter 1")
elif hyperparameters == 'arno_optimal':
model_params = io.loadmat('/home/agrigori/Programming/python/Sparse GP/solin-2014-supplement/arno_opt_params_to_python.mat')
var_1_per = 1
ls_1_per = 1 / 2 # division by 2 is done because meanings in parameters are different
period = 1
kernel1 = GPy.kern.sde_StdPeriodic(1, variance=var_1_per, lengthscale=ls_1_per, period=period, balance=False, approx_order = 6)
var_1_per = 1 # does not change
ls_1_per = 1
kernel2 = GPy.kern.sde_Matern32(1, variance=var_1_per, lengthscale=ls_1_per)
# Short term fluctuations kernel
var_1_per = 1
ls_1_per = 1
kernel3 = GPy.kern.sde_Matern32(1, variance=var_1_per, lengthscale=ls_1_per)
# RBF kernel. Not used if the parameter without_rbf = True
var_1_per = 1
ls_1_per = 1
kernel4 = GPy.kern.sde_RBF(1, variance=var_1_per, lengthscale=ls_1_per, balance=False, approx_order = 6)
# RBF kernel. Not used if the parameter
kernel = kernel1 * kernel2 + kernel3 + kernel4
if p_model == 'ss':
model = GPy.models.StateSpace( x_data, y_data, kernel=kernel, noise_var=1.0, balance=False, kalman_filter_type = 'regular')
elif p_model == 'sparse':
# Regularization assume regularization type 2
model = ss_sparse_model.SparcePrecisionGP(x_data,y_data,kernel, noise_var=1.0, balance=False, largest_cond_num=1e+15, regularization_type=2)
# Parameters of runs:
# optim bfgs: Kernel: std_per approx order=6 bal=False, rbf_approx_order=6, bal=False. Model: balance=False, largest_cond_num=1e+15, regularization_type=2
# optim scg: Kernel: std_per approx order=6 bal=False, rbf_approx_order=6, bal=False. Model: balance=False, largest_cond_num=1e+13, regularization_type=2
else:
raise ValueError("Wrong Parameter 2")
tt = np.array( [ model_params['per_magnSigma2'], model_params['per_period'], model_params['per_lengthscale']/2,
1.0, model_params['quasi_per_mat32_lengthscale'], model_params['mat32_inacc_magnSigma2'],
model_params['mat32_inacc_lengthScale'],
model_params['rbf_magnSigma2'], model_params['rbf_lengthscale'], model_params['opt_noise'] ] )
#print(tt)
model.param_array[:] = tt
#import pdb; pdb.set_trace()
model.kern.mul.Mat32.variance.fix()
model.kern.mul.std_periodic.period.fix()
if bound_qp_var:
model.kern.mul.std_periodic.lengthscale.constrain_bounded(0.2, 20000)
#model.kern.mul.std_periodic.
print(model)
class observer(object):
def __init__(self):
"""
"""
self.opt_obj_grads = None
self.opt_gradients = None
self.opt_params = None
self.opt_obj_funcs = None
def notif_callable(self, me, which=None):
"""
Description
"""
#import pdb; pdb.set_trace()
# me.obj_grads
if isinstance(self.opt_obj_grads , np.ndarray): # previous array
self.opt_obj_grads = np.vstack( (self.opt_obj_grads, me.obj_grads) )
else:
rr = me.obj_grads
if isinstance(self.opt_obj_grads , list):
if isinstance(rr, np.ndarray):
tt = np.empty( (len(self.opt_obj_grads), rr.shape[0]) ) * np.nan
self.opt_obj_grads = np.vstack( (tt, rr) )
else:
self.opt_obj_grads.append( (rr if rr is not None else np.nan) )
else:
if isinstance(rr , np.ndarray):
self.opt_obj_grads = rr
else:
self.opt_obj_grads = [ (rr if (rr is not None) else np.nan), ]
# me.gradient
if isinstance(self.opt_gradients , np.ndarray): # previous array
self.opt_gradients = np.vstack( (self.opt_gradients, me.gradient) )
else:
rr = me.gradient # same as me.objective_function_gradients()
if isinstance(self.opt_gradients , list):
if isinstance(rr, np.ndarray):
tt = np.empty( (len(self.opt_gradients), rr.shape[0]) ) * np.nan
self.opt_gradients = np.vstack( (tt, rr) )
else:
self.opt_gradients.append( (rr if rr is not None else np.nan) )
else:
if isinstance(rr , np.ndarray):
self.opt_gradients = rr
else:
self.opt_gradients = [ (rr if (rr is not None) else np.nan), ]
# me.param_array
if isinstance(self.opt_params , np.ndarray): # previous array
self.opt_params = np.vstack( (self.opt_params, me.param_array) )
else:
rr = me.param_array # same as me.objective_function_gradients()
if isinstance(self.opt_params , list):
if isinstance(rr, np.ndarray):
tt = np.empty( (len(self.opt_params), rr.shape[0]) ) * np.nan
self.opt_params = np.vstack( (tt, rr) )
else:
self.opt_params.append( (rr if rr is not None else np.nan) )
else:
if isinstance(rr , np.ndarray):
self.opt_params = rr
else:
self.opt_params = [ (rr if (rr is not None) else np.nan), ]
if self.opt_obj_funcs is None: # first iteration
self.opt_obj_funcs = [me.objective_function(),] if (me.objective_function() is not None) else [np.nan,]
else:
self.opt_obj_funcs.append( me.objective_function() )
def save(self, optim_save_path, optim_file_name, save= False):
"""
Saves optim data to file
"""
result_dict = {}
result_dict['opt_obj_grads'] = self.opt_obj_grads
result_dict['opt_gradients'] = self.opt_gradients
result_dict['opt_params'] = self.opt_params
result_dict['opt_obj_funcs'] = np.array(self.opt_obj_funcs)
optim_file_name = optim_file_name + '__optim_history'
if save:
io.savemat(os.path.join(optim_save_path, optim_file_name), result_dict)
return result_dict
oo = observer()
model.add_observer( oo, oo.notif_callable )
if optim == 'bfgs':
# L-BFGS-B optimization ->
#' factr' - criteria for stoping. If eps * factr < (f_k - f_(k+1)) then stop
# default: 1e7
#
# pgtol : float, optional
# The iteration will stop when
# ``max{|proj g_i | i = 1, ..., n} <= pgtol``
# where ``pg_i`` is the i-th component of the projected gradient.
# default: 1e-5
model.optimize(optimizer ='lbfgsb', bfgs_factor = 1e12, gtol=1e-3, messages=True)
# L-BFGS-B optimization <-
elif optim == 'scg':
# SCG optimization ->
# xtol: controls reduction in step size
# default: 1e-6
# ftol: controls reduction in objective
# default: 1e-6
# gtol: controls reduction in gradients
# default: 1e-5
model.optimize(optimizer ='scg', xtol = 1e-3, ftol=1e-3, gtol = 1e-3, messages=True)
# SCG optimization <-
else:
raise ValueError("Wrong Optimizer name")
#model.optimize(optimizer ='scg', messages=True)
print(model)
result_dict = {}
result_dict['params'] = model.param_array[:]
result_dict.update( oo.save('', '', save= False) )
save_file_name = '1_ex6_new_' + p_model+ '_' + hyperparameters + '_' + optim + '_' + '_qp_bound_' +str(bound_qp_var)
print(results_filex_prefix)
print(save_file_name)
io.savemat(os.path.join(results_filex_prefix, save_file_name), result_dict)
return model, oo
def experiment6_action3(hyperparameters='my_ex6_optimal'):
"""
Copies the first submision experiment6 action 4. It was working improperly
earlier. What is now?
'my_ex6_optimal' are probably somehow bad, did not work ok, however did not
try much as well.
'arno_optimal' - works well. Values of max_cond_number start from 1e+15. (1e+16 gives error)
1e+14 - visually indistinguishable from ss run. If we decrease
consitional number then variance expands, mean visually is not affected.
"""
import GPy.models.ss_sparse_model as ss_sparse_model
# Data ->
data_file_path= '/home/agrigori/Programming/python/Sparse GP/CO2_data/co2_weekly_init_clean.csv'
results_filex_prefix = '/home/agrigori/Programming/python/Sparse GP/Experiemnts/Results'
data = np.loadtxt(data_file_path);
data = data[ np.where(data[:,1] > 0)[0] ,:] # get rid of missing values
y_data = data[:,1]; y_data.shape = (y_data.shape[0],1)
x_data = data[:,0]; x_data.shape = (x_data.shape[0],1)
y_data = (y_data - np.mean(y_data)) / np.std(y_data)
x_data = x_data - 1974
# Data <-
if hyperparameters == 'my_ex6_optimal':
var_1_trend = 1.0
ls_1_trend = 200
per_per = 1
per_var = 1.0 # fixed
per_ls = 1
var_2_quasi = 1.0
ls_2_quasi = 1.0
var_3_quasi = 1.0
ls_3_quasi = 100.0
noise_var = 0.1
kernel = GPy.kern.sde_RBF(1, variance=var_1_trend, lengthscale=ls_1_trend) +\
GPy.kern.sde_StdPeriodic(1, variance=per_var, period = per_per, lengthscale=per_ls)*\
GPy. kern.sde_Matern32(1, variance=var_2_quasi, lengthscale=ls_2_quasi) +\
GPy. kern.sde_Matern32(1, variance=var_3_quasi, lengthscale=ls_3_quasi)
kernel.mul.std_periodic.variance.fix()
kernel.mul.std_periodic.period.fix()
result_dict = io.loadmat(os.path.join(results_filex_prefix,'ex6_params'))
params = result_dict['params'].squeeze()
kernel[:] = params[0:-1]
noise_var = params[-1]
model = ss_sparse_model.SparcePrecisionGP(x_data,y_data,kernel, noise_var=noise_var, balance=False, p_Inv_jitter=1e-14 )
elif hyperparameters == 'arno_optimal':
model_params = io.loadmat('/home/agrigori/Programming/python/Sparse GP/solin-2014-supplement/arno_opt_params_to_python.mat')
var_1_per = 1
ls_1_per = 1 / 2 # division by 2 is done because meanings in parameters are different
period = 1
kernel1 = GPy.kern.sde_StdPeriodic(1, variance=var_1_per, lengthscale=ls_1_per, period=period, balance=False, approx_order = 6)
var_1_per = 1 # does not change
ls_1_per = 1
kernel2 = GPy.kern.sde_Matern32(1, variance=var_1_per, lengthscale=ls_1_per)
# Short term fluctuations kernel
var_1_per = 1
ls_1_per = 1
kernel3 = GPy.kern.sde_Matern32(1, variance=var_1_per, lengthscale=ls_1_per)
# RBF kernel. Not used if the parameter without_rbf = True
var_1_per = 1
ls_1_per = 1
kernel4 = GPy.kern.sde_RBF(1, variance=var_1_per, lengthscale=ls_1_per, balance=False, approx_order = 6)
# RBF kernel. Not used if the parameter
kernel = kernel1 * kernel2 + kernel3 + kernel4
model = ss_sparse_model.SparcePrecisionGP(x_data,y_data,kernel, noise_var=1.0, balance=False, largest_cond_num=1e+14, regularization_type=2)
model.param_array[:] = np.array( [ model_params['per_magnSigma2'], model_params['per_period'], model_params['per_lengthscale']/2,
1.0, model_params['quasi_per_mat32_lengthscale'], model_params['mat32_inacc_magnSigma2'],
model_params['mat32_inacc_lengthScale'],
model_params['rbf_magnSigma2'], model_params['rbf_lengthscale'], model_params['opt_noise'] ] )
print(model)
# predict ->
years_to_predict = 8
step = np.mean( np.diff(x_data[:,0]) )
x_new = x_data[-1,0] + np.arange( step, years_to_predict, step ); x_new.shape = (x_new.shape[0],1)
x_new = np.vstack( (x_data, x_new)) # combine train and test data
ssm_mean, ssm_var = model.predict(x_new, include_likelihood=False, largest_cond_num=1e+14, regularization_type=2)
# predict <-
plt.figure(2)
#plt.title('Electricity Consumption Data', fontsize=30)
plt.plot( x_data, y_data, 'g-', label='Data',linewidth=1, markersize=5)
plt.plot( x_new, ssm_mean, 'b-', label='Data',linewidth=1, markersize=5)
plt.plot( x_new, ssm_mean+np.sqrt(ssm_var), 'r--', label='Data',linewidth=1, markersize=5)
plt.plot( x_new, ssm_mean-np.sqrt(ssm_var), 'r--', label='Data',linewidth=1, markersize=5)
#plt.xlabel('Time (Hours)', fontsize=25)
#plt.ylabel('Normalized Value', fontsize=25)
#plt.legend(loc=2)
plt.show()