def __init__(self): super(GPC, self).__init__() self.meanfunc = mean.Zero() # default prior mean self.covfunc = cov.RBF() # default prior covariance self.likfunc = lik.Erf() # erf likihood self.inffunc = inf.EP() # default inference method self.optimizer = opt.Minimize(self) # default optimizer
def useInference(self, newInf): if newInf == "Laplace": self.inffunc = inf.Laplace() elif newInf == "EP": self.inffunc = inf.EP() else: raise Exception('Possible inf values are "Laplace", "EP".')
def useLikelihood(self, newLik): ''' Use another likelihood function other than default Gaussian likelihood. :param str newLik: 'Laplace' ''' if newLik == "Laplace": self.likfunc = lik.Laplace() self.inffunc = inf.EP() else: raise Exception('Possible lik values are "Laplace".')
def useInference(self, newInf): ''' Use another inference techinique other than default exact inference. :param str newInf: 'Laplace' or 'EP' ''' if newInf == "Laplace": self.inffunc = inf.Laplace() elif newInf == "EP": self.inffunc = inf.EP() else: raise Exception('Possible inf values are "Laplace", "EP".')
def useLikelihood(self, newLik): if newLik == "Laplace": self.likfunc = lik.Laplace() self.inffunc = inf.EP() else: raise Exception('Possible lik values are "Laplace".')
def proceed(self, y=None, mu=None, s2=None, inffunc=None, der=None, nargout=1): sn = np.exp(self.hyp); b = sn/np.sqrt(2); if y == None: y = np.zeros_like(mu) if inffunc == None: # prediction mode if inf is not present if y == None: y = np.zeros_like(mu) s2zero = True; if not s2 == None: if np.linalg.norm(s2)>0: s2zero = False # s2==0? if s2zero: # log probability evaluation lp = -np.abs(y-mu)/b -np.log(2*b); s2 = 0 else: # prediction lp = self.proceed(y, mu, s2, inf.EP()) if nargout>1: ymu = mu # first y moment if nargout>2: ys2 = s2 + sn**2 # second y moment return lp,ymu,ys2 else: return lp,ymu else: return lp else: # inference mode if isinstance(inffunc, inf.Laplace): if der == None: # no derivative mode if y == None: y = np.zeros_like(mu) ymmu = y-mu lp = np.abs(ymmu)/b - np.log(2*b) if nargout>1: # derivative of log likelihood dip = np.sign(ymmu)/b if nargout>2: # 2nd derivative of log likelihood d2lp = np.zeros_like(ymmu) if nargout>3: # 3rd derivative of log likelihood d3lp = np.zeros_like(ymmu) return lp,dlp,d2lp,d3lp else: return lp,dlp,d2lp else: return lp,dlp else: return lp else: # derivative mode return [] # derivative w.r.t. hypers elif isinstance(inffunc, inf.EP): n = np.max([len(y.flatten()),len(mu.flatten()),len(s2.flatten()),len(sn.flatten())]) on = np.ones((n,1)) y = y*on; mu = mu*on; s2 = s2*on; sn = sn*on; fac = 1e3; # factor between the widths of the two distributions ... # ... from when one considered a delta peak, we use 3 orders of magnitude #idlik = np.reshape( (fac*sn) < np.sqrt(s2) , (sn.shape[0],) ) # Likelihood is a delta peak #idgau = np.reshape( (fac*np.sqrt(s2)) < sn , (sn.shape[0],) ) # Gaussian is a delta peak idlik = (fac*sn) < np.sqrt(s2) idgau = (fac*np.sqrt(s2)) < sn id = np.logical_and(np.logical_not(idgau),np.logical_not(idlik)) # interesting case in between if der == None: # no derivative mode lZ = np.zeros((n,1)) dlZ = np.zeros((n,1)) d2lZ = np.zeros((n,1)) if np.any(idlik): l = Gauss(log_sigma=np.log(s2[idlik])/2) a = l.proceed(mu[idlik], y[idlik]) lZ[idlik] = a[0]; dlZ[idlik] = a[1]; d2lZ[idlik] = a[2] if np.any(idgau): l = Laplace(log_hyp=np.log(sn[idgau])) a = l.proceed(mu=mu[idgau], y=y[idgau]) lZ[idgau] = a[0]; dlZ[idgau] = a[1]; d2lZ[idgau] = a[2] if np.any(id): # substitution to obtain unit variance, zero mean Laplacian tvar = s2[id]/(sn[id]**2+1e-16) tmu = (mu[id]-y[id])/(sn[id]+1e-16) # an implementation based on logphi(t) = log(normcdf(t)) zp = (tmu+np.sqrt(2)*tvar)/np.sqrt(tvar) zm = (tmu-np.sqrt(2)*tvar)/np.sqrt(tvar) ap = self._logphi(-zp)+np.sqrt(2)*tmu am = self._logphi( zm)-np.sqrt(2)*tmu apam = np.vstack((ap,am)).T lZ[id] = self._logsum2exp(apam) + tvar - np.log(sn[id]*np.sqrt(2.)) if nargout>1: lqp = -0.5*zp**2 - 0.5*np.log(2*np.pi) - self._logphi(-zp); # log( N(z)/Phi(z) ) lqm = -0.5*zm**2 - 0.5*np.log(2*np.pi) - self._logphi( zm); dap = -np.exp(lqp-0.5*np.log(s2[id])) + np.sqrt(2)/sn[id] dam = np.exp(lqm-0.5*np.log(s2[id])) - np.sqrt(2)/sn[id] _z1 = np.vstack((ap,am)).T _z2 = np.vstack((dap,dam)).T _x = np.array([[1],[1]]) dlZ[id] = self._expABz_expAx(_z1, _x, _z2, _x) if nargout>2: a = np.sqrt(8.)/sn[id]/np.sqrt(s2[id]); bp = 2./sn[id]**2 - (a - zp/s2[id])*np.exp(lqp) bm = 2./sn[id]**2 - (a + zm/s2[id])*np.exp(lqm) _x = np.reshape(np.array([1,1]),(2,1)) _z1 = np.reshape(np.array([ap,am]),(1,2)) _z2 = np.reshape(np.array([bp,bm]),(1,2)) d2lZ[id] = self._expABz_expAx(_z1, _x, _z2, _x) - dlZ[id]**2 return lZ,dlZ,d2lZ else: return lZ,dlZ else: return lZ else: # derivative mode dlZhyp = np.zeros((n,1)) if np.any(idlik): dlZhyp[idlik] = 0 if np.any(idgau): l = Laplace(log_hyp=np.log(sn[idgau])) a = l.proceed(mu=mu[idgau], y=y[idgau], inffunc='inf.Laplace', nargout=1) dlZhyp[idgau] = a[0] if np.any(id): # substitution to obtain unit variance, zero mean Laplacian tmu = (mu[id]-y[id])/(sn[id]+1e-16); tvar = s2[id]/(sn[id]**2+1e-16) zp = (tvar+tmu/np.sqrt(2))/np.sqrt(tvar); vp = tvar+np.sqrt(2)*tmu zm = (tvar-tmu/np.sqrt(2))/np.sqrt(tvar); vm = tvar-np.sqrt(2)*tmu dzp = (-s2[id]/sn[id]+tmu*sn[id]/np.sqrt(2)) / np.sqrt(s2[id]) dvp = -2*tvar - np.sqrt(2)*tmu dzm = (-s2[id]/sn[id]-tmu*sn[id]/np.sqrt(2)) / np.sqrt(s2[id]) dvm = -2*tvar + np.sqrt(2)*tmu lezp = self._lerfc(zp); # ap = exp(vp).*ezp lezm = self._lerfc(zm); # am = exp(vm).*ezm vmax = np.max(np.array([vp+lezp,vm+lezm]),axis=0); # subtract max to avoid numerical pb ep = np.exp(vp+lezp-vmax) em = np.exp(vm+lezm-vmax) dap = ep*(dvp - 2/np.sqrt(np.pi)*np.exp(-zp**2-lezp)*dzp) dam = em*(dvm - 2/np.sqrt(np.pi)*np.exp(-zm**2-lezm)*dzm) dlZhyp[id] = (dap+dam)/(ep+em) - 1; return dlZhyp # deriv. wrt hyp.lik elif isinstance(inffunc, inf.VB): n = len(s2.flatten()); b = np.zeros((n,1)); y = y*np.ones((n,1)); z = y return b,z
def proceed(self, y=None, mu=None, s2=None, inffunc=None, der=None, nargout=1): if not y == None: y = np.sign(y) y[y==0] = 1 else: y = 1 # allow only +/- 1 values if inffunc == None: # prediction mode if inf is not present y = y*np.ones_like(mu) # make y a vector s2zero = True; if not s2 == None: if np.linalg.norm(s2)>0: s2zero = False # s2==0? if s2zero: # log probability evaluation p,lp = self.cumGauss(y,mu,2) else: # prediction lp = self.proceed(y, mu, s2, inf.EP()) p = np.exp(lp) if nargout>1: ymu = 2*p-1 # first y moment if nargout>2: ys2 = 4*p*(1-p) # second y moment return lp,ymu,ys2 else: return lp,ymu else: return lp else: # inference mode if isinstance(inffunc, inf.Laplace): if der == None: # no derivative mode f = mu; yf = y*f # product latents and labels p,lp = self.cumGauss(y,f,2) if nargout>1: # derivative of log likelihood n_p = self.gauOverCumGauss(yf,p) dlp = y*n_p # derivative of log likelihood if nargout>2: # 2nd derivative of log likelihood d2lp = -n_p**2 - yf*n_p if nargout>3: # 3rd derivative of log likelihood d3lp = 2*y*n_p**3 + 3*f*n_p**2 + y*(f**2-1)*n_p return lp,dlp,d2lp,d3lp else: return lp,dlp,d2lp else: return lp,dlp else: return lp else: # derivative mode return [] # derivative w.r.t. hypers elif isinstance(inffunc, inf.EP): if der == None: # no derivative mode z = mu/np.sqrt(1+s2) junk,lZ = self.cumGauss(y,z,2) # log part function if not y == None: z = z*y if nargout>1: if y == None: y = 1 n_p = self.gauOverCumGauss(z,np.exp(lZ)) dlZ = y*n_p/np.sqrt(1.+s2) # 1st derivative wrt mean if nargout>2: d2lZ = -n_p*(z+n_p)/(1.+s2) # 2nd derivative wrt mean return lZ,dlZ,d2lZ else: return lZ,dlZ else: return lZ else: # derivative mode return [] # deriv. wrt hyp.lik '''
def proceed(self, y=None, mu=None, s2=None, inffunc=None, der=None, nargout=1): sn2 = np.exp(2. * self.hyp[0]) if inffunc == None: # prediction mode if y == None: y = np.zeros_like(mu) s2zero = True if (not s2==None) and np.linalg.norm(s2) > 0: s2zero = False if s2zero: # log probability lp = -(y-mu)**2 /sn2/2 - np.log(2.*np.pi*sn2)/2. s2 = np.zeros_like(s2) else: inf_func = inf.EP() # prediction lp = self.proceed(y, mu, s2, inf_func) if nargout>1: ymu = mu # first y moment if nargout>2: ys2 = s2 + sn2 # second y moment return lp,ymu,ys2 else: return lp,ymu else: return lp else: if isinstance(inffunc, inf.EP): if der == None: # no derivative mode lZ = -(y-mu)**2/(sn2+s2)/2. - np.log(2*np.pi*(sn2+s2))/2. # log part function if nargout>1: dlZ = (y-mu)/(sn2+s2) # 1st derivative w.r.t. mean if nargout>2: d2lZ = -1/(sn2+s2) # 2nd derivative w.r.t. mean return lZ,dlZ,d2lZ else: return lZ,dlZ else: return lZ else: # derivative mode dlZhyp = ((y-mu)**2/(sn2+s2)-1) / (1+s2/sn2) # deriv. w.r.t. hyp.lik return dlZhyp elif isinstance(inffunc, inf.Laplace): if der == None: # no derivative mode if y == None: y=0 ymmu = y-mu lp = -ymmu**2/(2*sn2) - np.log(2*np.pi*sn2)/2. if nargout>1: dlp = ymmu/sn2 # dlp, derivative of log likelihood if nargout>2: # d2lp, 2nd derivative of log likelihood d2lp = -np.ones_like(ymmu)/sn2 if nargout>3: # d3lp, 3rd derivative of log likelihood d3lp = np.zeros_like(ymmu) return lp,dlp,d2lp,d3lp else: return lp,dlp,d2lp else: return lp,dlp else: return lp else: # derivative mode lp_dhyp = (y-mu)**2/sn2 - 1 # derivative of log likelihood w.r.t. hypers dlp_dhyp = 2*(mu-y)/sn2 # first derivative, d2lp_dhyp = 2*np.ones_like(mu)/sn2 # and also of the second mu derivative return lp_dhyp,dlp_dhyp,d2lp_dhyp '''