def run_demo():
LG.basicConfig(level=LG.DEBUG)
SP.random.seed(10)
#1. create toy data
x,y,z,sigma,X,actual_inv,L = create_toy_data()
n_dimensions = 1
n_terms = 3
# build GP
likelihood = GaussLikISO()
covar_parms = SP.log([1,1,1E-5])
hyperparams = {'covar':covar_parms,'lik':SP.log([sigma]), 'warping': (1E-2*SP.random.randn(n_terms,3))}
SECF = se.SqexpCFARD(n_dimensions=n_dimensions)
covar = combinators.SumCF([SECF,BiasCF(n_dimensions=n_dimensions)])
warping_function = TanhWarpingFunction(n_terms=n_terms)
mean_function = LinMeanFunction(X= SP.ones([x.shape[0],1]))
hyperparams['mean'] = 1E-2* SP.randn(1)
bounds = {}
bounds.update(warping_function.get_bounds())
gp = WARPEDGP(warping_function = warping_function,
mean_function = mean_function,
covar_func=covar, likelihood=likelihood, x=x, y=z)
opt_model_params = opt_hyper(gp, hyperparams,
bounds = bounds,
gradcheck=True)[0]
print "WARPED GP (neg) likelihood: ", gp.LML(hyperparams)
#hyperparams['mean'] = SP.log(1)
PL.figure()
PL.plot(z)
PL.plot(warping_function.f(y,hyperparams['warping']))
PL.legend(["real function", "larnt function"])
PL.figure()
PL.plot(actual_inv)
PL.plot(warping_function.f_inv(gp.y,hyperparams['warping']))
PL.legend(['real inverse','learnt inverse'])
hyperparams.pop("warping")
hyperparams.pop("mean")
gp = GP(covar,likelihood=likelihood,x=x,y=y)
opt_model_params = opt_hyper(gp,hyperparams,
gradcheck=False)[0]
print "GP (neg) likelihood: ", gp.LML(hyperparams)
def optimize(self, initial_hyperparams,
bounds=None,
burniniter=200,
gradient_tolerance=1e-15,
maxiter=15E3,
release_beta=True):
ifilter = self.get_filter(initial_hyperparams)
for key in ifilter.iterkeys():
if key.startswith('beta'):
ifilter[key][:] = 0
# fix beta for burn in:
hyperparams, _ = opt_hyper(self, initial_hyperparams,
gradient_tolerance=gradient_tolerance, maxiter=burniniter,
Ifilter=ifilter, bounds=bounds)
if(release_beta):
hyperparams, _ = opt_hyper(self, hyperparams,
gradient_tolerance=gradient_tolerance,
bounds=bounds, maxiter=maxiter)
return hyperparams
def predict_model_likelihoods(self, training_data=None, interval_indices=get_model_structure(), *args, **kwargs):
"""
Predict the probabilities of the models (individual and common) to describe the data.
It will optimize hyperparameters respectively.
**Parameters**:
training_data : dict traning_data
The training data to learn from. Input are time-values and
output are expression-values of e.g. a timeseries.
If not given, training data must be given previously by
:py:class:`gptwosample.twosample.basic.set_data`.
interval_indices: :py:class:`gptwosample.data.data_base.get_model_structure()`
interval indices, which assign data to individual or common model,
respectively.
args : [..]
see :py:class:`pygp.gpr.gp_base.GP`
kwargs : {..}
see :py:class:`pygp.gpr.gp_base.GP`
"""
if(training_data is not None):
self.set_data(training_data)
for name, model in self._models.iteritems():
model.set_active_set_indices(interval_indices[name])
try:
if(self._learn_hyperparameters):
opt_hyperparameters = opt_hyper(model,
self._initial_hyperparameters[name],
priors=self._priors[name],
*args, **kwargs)[0]
self._learned_hyperparameters[name] = opt_hyperparameters
else:
self._learned_hyperparameters[name] = self._initial_hyperparameters[name]
except ValueError as r:
print "caught error:", r.message, "\r",
self._learned_hyperparameters[name] = self._initial_hyperparameters[name]
self._model_likelihoods[name] = model.LML(self._learned_hyperparameters[name],
priors=self._priors)
return self._model_likelihoods
# psi_1 = covar.psi_1(hyperparams['covar'], mean, variance, inducing_variables)
# psi_2 = covar.psi_2(hyperparams['covar'], mean, variance, inducing_variables)
#
#go through all hyperparams and build bound array (flattened)
# learn latent variables
skeys = numpy.sort(hyperparams.keys())
param_struct = dict([(name,hyperparams[name].shape) for name in skeys])
bounds = {}#{'beta': [(numpy.log(1./1E-3))]}
hyper_for_optimizing = copy.deepcopy(hyperparams)
#hyper_for_optimizing[vars_id] = numpy.log(hyper_for_optimizing[vars_id])
opt_hyperparams, opt_lml = opt_hyper(g, hyper_for_optimizing, bounds=bounds, maxiter=10000, messages=True)
#opt_hyperparams[vars_id] = numpy.exp(opt_hyperparams[vars_id])
# opt_covar = numpy.exp(2*opt_hyperparams['covar'])
# param_diffs = numpy.abs(numpy.subtract.outer(opt_covar,opt_covar)).round(5)
# while (param_diffs < .1).all():
## pylab.close('all')
## pylab.bar(range(Qlearn),numpy.exp(2*opt_hyperparams['covar']))
## pylab.figure()
## opt_covar = numpy.exp(2*opt_hyperparams['covar'])
## s = numpy.arange(0,Qlearn)[opt_hyperparams['covar']==opt_hyperparams['covar'].max()][0]
## for s in xrange(Qlearn):
## pylab.plot(signals[:,s],'--o')
## pylab.errorbar(range(N), opt_hyperparams[mean_id][:,s], yerr=numpy.sqrt(opt_hyperparams[vars_id])[:,s], ls='-', alpha=1./numpy.sqrt(opt_hyperparams[vars_id])[:,s].max())
# yn = 'y' #raw_input("want to restart? [y]/n: ")
# if len(yn) < 1:
# yn = 'y'
X = Spca.copy()
#X = SP.random.randn(N,K)
gplvm = GPLVM(covar_func=covariance, x=X, y=Y)
gpr = GP(covar_func=covariance, x=X, y=Y[:, 0])
#construct hyperparams
covar = SP.log([0.1, 1.0, 0.1])
#X are hyperparameters, i.e. we optimize over them also
#1. this is jointly with the latent X
X_ = X.copy()
hyperparams = {'covar': covar, 'x': X_}
#for testing just covar params alone:
#hyperparams = {'covar': covar}
#evaluate log marginal likelihood
lml = gplvm.LML(hyperparams=hyperparams)
[opt_model_params, opt_lml] = opt_hyper(gplvm, hyperparams, gradcheck=False)
Xo = opt_model_params['x']
for k in xrange(K):
print SP.corrcoef(Spca[:, k], S[:, k])
print "=================="
for k in xrange(K):
print SP.corrcoef(Xo[:, k], S[:, k])
X = Spca.copy()
#X = SP.random.randn(N,K)
gplvm = GPLVM(covar_func=covariance, x=X, y=Y)
gpr = GP(covar_func=covariance, x=X, y=Y[:, 0])
#construct hyperparams
covar = SP.log([0.1, 1.0, 0.1])
#X are hyperparameters, i.e. we optimize over them also
#1. this is jointly with the latent X
X_ = X.copy()
hyperparams = {'covar': covar, 'x': X_}
#for testing just covar params alone:
#hyperparams = {'covar': covar}
#evaluate log marginal likelihood
lml = gplvm.LML(hyperparams=hyperparams)
[opt_model_params, opt_lml] = opt_hyper(gplvm,
hyperparams,
gradcheck=False)
Xo = opt_model_params['x']
for k in xrange(K):
print SP.corrcoef(Spca[:, k], S[:, k])
print "=================="
for k in xrange(K):
print SP.corrcoef(Xo[:, k], S[:, k])
return gp.LML(hyperparams,)
def df4(x):
hyperparams['warping'][:] = x
return gp.LMLgrad(hyperparams)['warping']
x = hyperparams['warping'].copy()
checkgrad(f4,df4,x)
lmld= gp.LMLgrad(hyperparams)
print lmld
#gp = GP(covar,likelihood=likelihood,x=x,y=y)
opt_model_params = opt_hyper(gp, hyperparams,
bounds = bounds,
maxiter=10000,
gradcheck=True)[0]
PL.figure(2)
z_values = SP.linspace(z.min(),z.max(),100)
PL.plot(Itrafo(gp.y))
#opt_hyperparamsPL.plot(z_values,Itrafo(L*z_values))
pred_inverse = warping_function.f_inv(gp.y,
opt_model_params['warping'],
iterations = 10)
PL.plot(pred_inverse)
#PL.plot(z,y,'r.')
#PL.legend(['real inverse','learnt inverse','data'])
hyperparams['warping'][:] = x
return gp.LML(hyperparams, )
def df4(x):
hyperparams['warping'][:] = x
return gp.LMLgrad(hyperparams)['warping']
x = hyperparams['warping'].copy()
checkgrad(f4, df4, x)
lmld = gp.LMLgrad(hyperparams)
print lmld
#gp = GP(covar,likelihood=likelihood,x=x,y=y)
opt_model_params = opt_hyper(gp,
hyperparams,
bounds=bounds,
maxiter=10000,
gradcheck=True)[0]
PL.figure(2)
z_values = SP.linspace(z.min(), z.max(), 100)
PL.plot(Itrafo(gp.y))
#opt_hyperparamsPL.plot(z_values,Itrafo(L*z_values))
pred_inverse = warping_function.f_inv(gp.y,
opt_model_params['warping'],
iterations=10)
PL.plot(pred_inverse)
#PL.plot(z,y,'r.')
#PL.legend(['real inverse','learnt inverse','data'])
covar_params = SP.random.randn(n_dimensions + 1)
lik_params = SP.random.randn(1)
Xs = SP.linspace(X.min() - 3, X.max() + 3)[:, SP.newaxis]
t0 = time.time()
#pygp: OLD
covar_ = se.SqexpCFARD(n_dimensions)
ll_ = lik.GaussLikISO()
hyperparams_ = {'covar': covar_params, 'lik': lik_params}
gp_ = GP.GP(covar_, likelihood=ll_, x=X, y=y)
lml_ = gp_.LML(hyperparams_)
dlml_ = gp_.LMLgrad(hyperparams_)
#optimize using pygp:
opt_params_ = opt.opt_hyper(gp_, hyperparams_)[0]
lmlo_ = gp_.LML(opt_params_)
pdb.set_trace()
#GPMIX:
cov = SP.ones([y.shape[0], 2])
cov[:, 1] = SP.randn(cov.shape[0])
covar = limix.CCovSqexpARD(n_dimensions)
ll = limix.CLikNormalIso()
if 1:
data = limix.CLinearMean(y, cov)
data_params = SP.ones([cov.shape[1]])
else:
data = limix.CData()
covar_params = SP.random.randn(n_dimensions+1)
lik_params = SP.random.randn(1)
Xs = SP.linspace(X.min()-3,X.max()+3)[:,SP.newaxis]
t0 = time.time()
#pygp: OLD
covar_ = se.SqexpCFARD(n_dimensions)
ll_ = lik.GaussLikISO()
hyperparams_ = {'covar':covar_params,'lik':lik_params}
gp_ = GP.GP(covar_,likelihood=ll_,x=X,y=y)
lml_ = gp_.LML(hyperparams_)
dlml_ = gp_.LMLgrad(hyperparams_)
#optimize using pygp:
opt_params_ = opt.opt_hyper(gp_,hyperparams_)[0]
lmlo_ = gp_.LML(opt_params_)
pdb.set_trace()
#GPMIX:
cov = SP.ones([y.shape[0],2])
cov[:,1] = SP.randn(cov.shape[0])
covar = limix.CCovSqexpARD(n_dimensions)
ll = limix.CLikNormalIso()
if 1:
data = limix.CLinearMean(y,cov);
data_params = SP.ones([cov.shape[1]])
def run_demo():
LG.basicConfig(level=LG.INFO)
PL.figure()
random.seed(1)
#0. generate Toy-Data; just samples from a superposition of a sin + linear trend
n_replicates = 4
xmin = 1
xmax = 2.5*SP.pi
x1_time_steps = 10
x2_time_steps = 20
x1 = SP.zeros(x1_time_steps*n_replicates)
x2 = SP.zeros(x2_time_steps*n_replicates)
for i in xrange(n_replicates):
x1[i*x1_time_steps:(i+1)*x1_time_steps] = SP.linspace(xmin,xmax,x1_time_steps)
x2[i*x2_time_steps:(i+1)*x2_time_steps] = SP.linspace(xmin,xmax,x2_time_steps)
C = 2 #offset
#b = 0.5
sigma1 = 0.15
sigma2 = 0.15
n_noises = 1
b = 0
y1 = b*x1 + C + 1*SP.sin(x1)
# dy1 = b + 1*SP.cos(x1)
y1 += sigma1*random.randn(y1.shape[0])
y1-= y1.mean()
y2 = b*x2 + C + 1*SP.sin(x2)
# dy2 = b + 1*SP.cos(x2)
y2 += sigma2*random.randn(y2.shape[0])
y2-= y2.mean()
for i in xrange(n_replicates):
x1[i*x1_time_steps:(i+1)*x1_time_steps] += .7 + (i/2.)
x2[i*x2_time_steps:(i+1)*x2_time_steps] -= .7 + (i/2.)
x1 = x1[:,SP.newaxis]
x2 = x2[:,SP.newaxis]
x = SP.concatenate((x1,x2),axis=0)
y = SP.concatenate((y1,y2),axis=0)
#predictions:
X = SP.linspace(xmin-n_replicates,xmax+n_replicates,100*n_replicates)[:,SP.newaxis]
#hyperparamters
dim = 1
replicate_indices = []
for i,xi in enumerate((x1,x2)):
for rep in SP.arange(i*n_replicates, (i+1)*n_replicates):
replicate_indices.extend(SP.repeat(rep,len(xi)/n_replicates))
replicate_indices = SP.array(replicate_indices)
n_replicates = len(SP.unique(replicate_indices))
logthetaCOVAR = [1,1]
logthetaCOVAR.extend(SP.repeat(SP.exp(1),n_replicates))
logthetaCOVAR.extend([sigma1])
logthetaCOVAR = SP.log(logthetaCOVAR)#,sigma2])
hyperparams = {'covar':logthetaCOVAR}
SECF = se.SqexpCFARD(dim)
#noiseCF = noise.NoiseReplicateCF(replicate_indices)
noiseCF = noise.NoiseCFISO()
shiftCF = combinators.ShiftCF(SECF,replicate_indices)
CovFun = combinators.SumCF((shiftCF,noiseCF))
covar_priors = []
#scale
covar_priors.append([lnpriors.lnGammaExp,[1,2]])
for i in range(dim):
covar_priors.append([lnpriors.lnGammaExp,[1,1]])
#shift
for i in range(n_replicates):
covar_priors.append([lnpriors.lnGauss,[0,.5]])
#noise
for i in range(n_noises):
covar_priors.append([lnpriors.lnGammaExp,[1,1]])
covar_priors = SP.array(covar_priors)
priors = {'covar':covar_priors}
Ifilter = {'covar': SP.ones(n_replicates+3)}
gpr = GP(CovFun,x=x,y=y)
opt_model_params = opt_hyper(gpr,hyperparams,priors=priors,gradcheck=True,Ifilter=Ifilter)[0]
#predict
[M,S_glu] = gpr.predict(opt_model_params,X)
T = opt_model_params['covar'][2:2+n_replicates]
PL.subplot(212)
gpr_plot.plot_sausage(X,M,SP.sqrt(S_glu),format_line=dict(alpha=1,color='g',lw=2, ls='-'))
gpr_plot.plot_training_data(x,y,shift=T,replicate_indices=replicate_indices,draw_arrows=2)
PL.suptitle("Example for GPTimeShift with simulated data", fontsize=23)
PL.title("Regression including time shift")
PL.xlabel("x")
PL.ylabel("y")
ylim = PL.ylim()
gpr = GP(combinators.SumCF((SECF,noiseCF)),x=x,y=y)
priors = {'covar':covar_priors[[0,1,-1]]}
hyperparams = {'covar':logthetaCOVAR[[0,1,-1]]}
opt_model_params = opt_hyper(gpr,hyperparams,priors=priors,gradcheck=True)[0]
PL.subplot(211)
#predict
[M,S_glu] = gpr.predict(opt_model_params,X)
gpr_plot.plot_sausage(X,M,SP.sqrt(S_glu),format_line=dict(alpha=1,color='g',lw=2, ls='-'))
gpr_plot.plot_training_data(x,y,replicate_indices=replicate_indices)
PL.title("Regression without time shift")
PL.xlabel("x")
PL.ylabel("y")
PL.ylim(ylim)
PL.subplots_adjust(left=.1, bottom=.1,
right=.96, top=.8,
wspace=.4, hspace=.4)
PL.show()