def __init__(self):
super(GPC, self).__init__()
self.meanfunc = mean.Zero() # default prior mean
self.covfunc = cov.RBF() # default prior covariance
self.likfunc = lik.Erf() # erf likihood
self.inffunc = inf.EP() # default inference method
self.optimizer = opt.Minimize(self) # default optimizer
def useInference(self, newInf):
if newInf == "Laplace":
self.inffunc = inf.Laplace()
elif newInf == "EP":
self.inffunc = inf.EP()
else:
raise Exception('Possible inf values are "Laplace", "EP".')
def useLikelihood(self, newLik):
'''
Use another likelihood function other than default Gaussian likelihood.
:param str newLik: 'Laplace'
'''
if newLik == "Laplace":
self.likfunc = lik.Laplace()
self.inffunc = inf.EP()
else:
raise Exception('Possible lik values are "Laplace".')
def useInference(self, newInf):
'''
Use another inference techinique other than default exact inference.
:param str newInf: 'Laplace' or 'EP'
'''
if newInf == "Laplace":
self.inffunc = inf.Laplace()
elif newInf == "EP":
self.inffunc = inf.EP()
else:
raise Exception('Possible inf values are "Laplace", "EP".')
def useLikelihood(self, newLik):
if newLik == "Laplace":
self.likfunc = lik.Laplace()
self.inffunc = inf.EP()
else:
raise Exception('Possible lik values are "Laplace".')
def proceed(self, y=None, mu=None, s2=None, inffunc=None, der=None, nargout=1):
sn = np.exp(self.hyp); b = sn/np.sqrt(2);
if y == None:
y = np.zeros_like(mu)
if inffunc == None: # prediction mode if inf is not present
if y == None:
y = np.zeros_like(mu)
s2zero = True;
if not s2 == None:
if np.linalg.norm(s2)>0:
s2zero = False # s2==0?
if s2zero: # log probability evaluation
lp = -np.abs(y-mu)/b -np.log(2*b); s2 = 0
else: # prediction
lp = self.proceed(y, mu, s2, inf.EP())
if nargout>1:
ymu = mu # first y moment
if nargout>2:
ys2 = s2 + sn**2 # second y moment
return lp,ymu,ys2
else:
return lp,ymu
else:
return lp
else: # inference mode
if isinstance(inffunc, inf.Laplace):
if der == None: # no derivative mode
if y == None:
y = np.zeros_like(mu)
ymmu = y-mu
lp = np.abs(ymmu)/b - np.log(2*b)
if nargout>1: # derivative of log likelihood
dip = np.sign(ymmu)/b
if nargout>2: # 2nd derivative of log likelihood
d2lp = np.zeros_like(ymmu)
if nargout>3: # 3rd derivative of log likelihood
d3lp = np.zeros_like(ymmu)
return lp,dlp,d2lp,d3lp
else:
return lp,dlp,d2lp
else:
return lp,dlp
else:
return lp
else: # derivative mode
return [] # derivative w.r.t. hypers
elif isinstance(inffunc, inf.EP):
n = np.max([len(y.flatten()),len(mu.flatten()),len(s2.flatten()),len(sn.flatten())])
on = np.ones((n,1))
y = y*on; mu = mu*on; s2 = s2*on; sn = sn*on;
fac = 1e3; # factor between the widths of the two distributions ...
# ... from when one considered a delta peak, we use 3 orders of magnitude
#idlik = np.reshape( (fac*sn) < np.sqrt(s2) , (sn.shape[0],) ) # Likelihood is a delta peak
#idgau = np.reshape( (fac*np.sqrt(s2)) < sn , (sn.shape[0],) ) # Gaussian is a delta peak
idlik = (fac*sn) < np.sqrt(s2)
idgau = (fac*np.sqrt(s2)) < sn
id = np.logical_and(np.logical_not(idgau),np.logical_not(idlik)) # interesting case in between
if der == None: # no derivative mode
lZ = np.zeros((n,1))
dlZ = np.zeros((n,1))
d2lZ = np.zeros((n,1))
if np.any(idlik):
l = Gauss(log_sigma=np.log(s2[idlik])/2)
a = l.proceed(mu[idlik], y[idlik])
lZ[idlik] = a[0]; dlZ[idlik] = a[1]; d2lZ[idlik] = a[2]
if np.any(idgau):
l = Laplace(log_hyp=np.log(sn[idgau]))
a = l.proceed(mu=mu[idgau], y=y[idgau])
lZ[idgau] = a[0]; dlZ[idgau] = a[1]; d2lZ[idgau] = a[2]
if np.any(id):
# substitution to obtain unit variance, zero mean Laplacian
tvar = s2[id]/(sn[id]**2+1e-16)
tmu = (mu[id]-y[id])/(sn[id]+1e-16)
# an implementation based on logphi(t) = log(normcdf(t))
zp = (tmu+np.sqrt(2)*tvar)/np.sqrt(tvar)
zm = (tmu-np.sqrt(2)*tvar)/np.sqrt(tvar)
ap = self._logphi(-zp)+np.sqrt(2)*tmu
am = self._logphi( zm)-np.sqrt(2)*tmu
apam = np.vstack((ap,am)).T
lZ[id] = self._logsum2exp(apam) + tvar - np.log(sn[id]*np.sqrt(2.))
if nargout>1:
lqp = -0.5*zp**2 - 0.5*np.log(2*np.pi) - self._logphi(-zp); # log( N(z)/Phi(z) )
lqm = -0.5*zm**2 - 0.5*np.log(2*np.pi) - self._logphi( zm);
dap = -np.exp(lqp-0.5*np.log(s2[id])) + np.sqrt(2)/sn[id]
dam = np.exp(lqm-0.5*np.log(s2[id])) - np.sqrt(2)/sn[id]
_z1 = np.vstack((ap,am)).T
_z2 = np.vstack((dap,dam)).T
_x = np.array([[1],[1]])
dlZ[id] = self._expABz_expAx(_z1, _x, _z2, _x)
if nargout>2:
a = np.sqrt(8.)/sn[id]/np.sqrt(s2[id]);
bp = 2./sn[id]**2 - (a - zp/s2[id])*np.exp(lqp)
bm = 2./sn[id]**2 - (a + zm/s2[id])*np.exp(lqm)
_x = np.reshape(np.array([1,1]),(2,1))
_z1 = np.reshape(np.array([ap,am]),(1,2))
_z2 = np.reshape(np.array([bp,bm]),(1,2))
d2lZ[id] = self._expABz_expAx(_z1, _x, _z2, _x) - dlZ[id]**2
return lZ,dlZ,d2lZ
else:
return lZ,dlZ
else:
return lZ
else: # derivative mode
dlZhyp = np.zeros((n,1))
if np.any(idlik):
dlZhyp[idlik] = 0
if np.any(idgau):
l = Laplace(log_hyp=np.log(sn[idgau]))
a = l.proceed(mu=mu[idgau], y=y[idgau], inffunc='inf.Laplace', nargout=1)
dlZhyp[idgau] = a[0]
if np.any(id):
# substitution to obtain unit variance, zero mean Laplacian
tmu = (mu[id]-y[id])/(sn[id]+1e-16); tvar = s2[id]/(sn[id]**2+1e-16)
zp = (tvar+tmu/np.sqrt(2))/np.sqrt(tvar); vp = tvar+np.sqrt(2)*tmu
zm = (tvar-tmu/np.sqrt(2))/np.sqrt(tvar); vm = tvar-np.sqrt(2)*tmu
dzp = (-s2[id]/sn[id]+tmu*sn[id]/np.sqrt(2)) / np.sqrt(s2[id])
dvp = -2*tvar - np.sqrt(2)*tmu
dzm = (-s2[id]/sn[id]-tmu*sn[id]/np.sqrt(2)) / np.sqrt(s2[id])
dvm = -2*tvar + np.sqrt(2)*tmu
lezp = self._lerfc(zp); # ap = exp(vp).*ezp
lezm = self._lerfc(zm); # am = exp(vm).*ezm
vmax = np.max(np.array([vp+lezp,vm+lezm]),axis=0); # subtract max to avoid numerical pb
ep = np.exp(vp+lezp-vmax)
em = np.exp(vm+lezm-vmax)
dap = ep*(dvp - 2/np.sqrt(np.pi)*np.exp(-zp**2-lezp)*dzp)
dam = em*(dvm - 2/np.sqrt(np.pi)*np.exp(-zm**2-lezm)*dzm)
dlZhyp[id] = (dap+dam)/(ep+em) - 1;
return dlZhyp # deriv. wrt hyp.lik
elif isinstance(inffunc, inf.VB):
n = len(s2.flatten()); b = np.zeros((n,1)); y = y*np.ones((n,1)); z = y
return b,z
def proceed(self, y=None, mu=None, s2=None, inffunc=None, der=None, nargout=1):
if not y == None:
y = np.sign(y)
y[y==0] = 1
else:
y = 1 # allow only +/- 1 values
if inffunc == None: # prediction mode if inf is not present
y = y*np.ones_like(mu) # make y a vector
s2zero = True;
if not s2 == None:
if np.linalg.norm(s2)>0:
s2zero = False # s2==0?
if s2zero: # log probability evaluation
p,lp = self.cumGauss(y,mu,2)
else: # prediction
lp = self.proceed(y, mu, s2, inf.EP())
p = np.exp(lp)
if nargout>1:
ymu = 2*p-1 # first y moment
if nargout>2:
ys2 = 4*p*(1-p) # second y moment
return lp,ymu,ys2
else:
return lp,ymu
else:
return lp
else: # inference mode
if isinstance(inffunc, inf.Laplace):
if der == None: # no derivative mode
f = mu; yf = y*f # product latents and labels
p,lp = self.cumGauss(y,f,2)
if nargout>1: # derivative of log likelihood
n_p = self.gauOverCumGauss(yf,p)
dlp = y*n_p # derivative of log likelihood
if nargout>2: # 2nd derivative of log likelihood
d2lp = -n_p**2 - yf*n_p
if nargout>3: # 3rd derivative of log likelihood
d3lp = 2*y*n_p**3 + 3*f*n_p**2 + y*(f**2-1)*n_p
return lp,dlp,d2lp,d3lp
else:
return lp,dlp,d2lp
else:
return lp,dlp
else:
return lp
else: # derivative mode
return [] # derivative w.r.t. hypers
elif isinstance(inffunc, inf.EP):
if der == None: # no derivative mode
z = mu/np.sqrt(1+s2)
junk,lZ = self.cumGauss(y,z,2) # log part function
if not y == None:
z = z*y
if nargout>1:
if y == None: y = 1
n_p = self.gauOverCumGauss(z,np.exp(lZ))
dlZ = y*n_p/np.sqrt(1.+s2) # 1st derivative wrt mean
if nargout>2:
d2lZ = -n_p*(z+n_p)/(1.+s2) # 2nd derivative wrt mean
return lZ,dlZ,d2lZ
else:
return lZ,dlZ
else:
return lZ
else: # derivative mode
return [] # deriv. wrt hyp.lik
'''
def proceed(self, y=None, mu=None, s2=None, inffunc=None, der=None, nargout=1):
sn2 = np.exp(2. * self.hyp[0])
if inffunc == None: # prediction mode
if y == None:
y = np.zeros_like(mu)
s2zero = True
if (not s2==None) and np.linalg.norm(s2) > 0:
s2zero = False
if s2zero: # log probability
lp = -(y-mu)**2 /sn2/2 - np.log(2.*np.pi*sn2)/2.
s2 = np.zeros_like(s2)
else:
inf_func = inf.EP() # prediction
lp = self.proceed(y, mu, s2, inf_func)
if nargout>1:
ymu = mu # first y moment
if nargout>2:
ys2 = s2 + sn2 # second y moment
return lp,ymu,ys2
else:
return lp,ymu
else:
return lp
else:
if isinstance(inffunc, inf.EP):
if der == None: # no derivative mode
lZ = -(y-mu)**2/(sn2+s2)/2. - np.log(2*np.pi*(sn2+s2))/2. # log part function
if nargout>1:
dlZ = (y-mu)/(sn2+s2) # 1st derivative w.r.t. mean
if nargout>2:
d2lZ = -1/(sn2+s2) # 2nd derivative w.r.t. mean
return lZ,dlZ,d2lZ
else:
return lZ,dlZ
else:
return lZ
else: # derivative mode
dlZhyp = ((y-mu)**2/(sn2+s2)-1) / (1+s2/sn2) # deriv. w.r.t. hyp.lik
return dlZhyp
elif isinstance(inffunc, inf.Laplace):
if der == None: # no derivative mode
if y == None:
y=0
ymmu = y-mu
lp = -ymmu**2/(2*sn2) - np.log(2*np.pi*sn2)/2.
if nargout>1:
dlp = ymmu/sn2 # dlp, derivative of log likelihood
if nargout>2: # d2lp, 2nd derivative of log likelihood
d2lp = -np.ones_like(ymmu)/sn2
if nargout>3: # d3lp, 3rd derivative of log likelihood
d3lp = np.zeros_like(ymmu)
return lp,dlp,d2lp,d3lp
else:
return lp,dlp,d2lp
else:
return lp,dlp
else:
return lp
else: # derivative mode
lp_dhyp = (y-mu)**2/sn2 - 1 # derivative of log likelihood w.r.t. hypers
dlp_dhyp = 2*(mu-y)/sn2 # first derivative,
d2lp_dhyp = 2*np.ones_like(mu)/sn2 # and also of the second mu derivative
return lp_dhyp,dlp_dhyp,d2lp_dhyp
'''
