def _set_expected_stats_latents(self, smoothed_mus, smoothed_sigmas): sub_pops = self.obs_scheme.sub_pops obs_pops = self.obs_scheme.obs_pops obs_time = self.obs_scheme.obs_time idx_grp = self.obs_scheme.idx_grp obs_idx = self.obs_scheme.obs_idx if obs_time.size > 1: T = np.zeros(len(idx_grp)) Ex = np.zeros(self.n) ExxT = np.zeros((self.n, self.n)) Exe = np.zeros((self.n, len(idx_grp))) ExxTj = np.zeros((self.n, self.n, len(idx_grp))) ExxTe = np.zeros((self.n + 1, self.n + 1, len(idx_grp))) for i in range(obs_time.size): ts = range(obs_time[0]) if i == 0 else range( obs_time[i - 1], obs_time[i]) x = smoothed_mus[ts, :] sx = np.sum(x, 0) sxxT = smoothed_sigmas[ts, :, :].sum(0) + x.T.dot(x) for j in obs_idx[i]: Exe[:, j] += sx ExxTj[:, :, j] += sxxT T[j] += len(ts) Ex += sx ExxT += sxxT for j in range(len(idx_grp)): ExxTe[:, :, j] = blockarray( [[ExxTj[:, :, j], np.atleast_2d(Exe[:, j]).T], [np.atleast_2d(Exe[:, j]), np.atleast_2d(T[j])]]) else: Ex = smoothed_mus.sum(0) ExxT = smoothed_sigmas.sum(0) + smoothed_mus.T.dot(smoothed_mus) E_xt_xtT = \ ExxT - (smoothed_sigmas[-1] + np.outer(smoothed_mus[-1],smoothed_mus[-1])) E_xtp1_xtp1T = \ ExxT - (smoothed_sigmas[0] + np.outer(smoothed_mus[0], smoothed_mus[0])) if self.model.emission_distn.affine: ExxT = blockarray([[ExxT, np.atleast_2d(Ex).T], [np.atleast_2d(Ex), np.atleast_2d(self.T)]]) if not obs_time.size > 1: ExxTe = ExxT return ExxT, E_xt_xtT, E_xtp1_xtp1T, ExxTe
def _set_expected_stats_latents(self, smoothed_mus, smoothed_sigmas): sub_pops = self.obs_scheme.sub_pops obs_pops = self.obs_scheme.obs_pops obs_time = self.obs_scheme.obs_time idx_grp = self.obs_scheme.idx_grp obs_idx = self.obs_scheme.obs_idx if obs_time.size > 1: T = np.zeros(len(idx_grp)) Ex = np.zeros(self.n) ExxT = np.zeros((self.n,self.n)) Exe = np.zeros((self.n, len(idx_grp))) ExxTj = np.zeros((self.n, self.n, len(idx_grp))) ExxTe = np.zeros((self.n+1, self.n+1, len(idx_grp))) for i in range(obs_time.size): ts = range(obs_time[0]) if i==0 else range(obs_time[i-1], obs_time[i]) x = smoothed_mus[ts,:] sx = np.sum(x,0) sxxT = smoothed_sigmas[ts,:,:].sum(0) + x.T.dot(x) for j in obs_idx[i]: Exe[:,j] += sx ExxTj[:,:,j] += sxxT T[j] += len(ts) Ex += sx ExxT += sxxT for j in range(len(idx_grp)): ExxTe[:,:,j] = blockarray([[ExxTj[:,:,j],np.atleast_2d(Exe[:,j]).T], [np.atleast_2d(Exe[:,j]),np.atleast_2d(T[j])]]) else: Ex = smoothed_mus.sum(0) ExxT = smoothed_sigmas.sum(0) + smoothed_mus.T.dot(smoothed_mus) E_xt_xtT = \ ExxT - (smoothed_sigmas[-1] + np.outer(smoothed_mus[-1],smoothed_mus[-1])) E_xtp1_xtp1T = \ ExxT - (smoothed_sigmas[0] + np.outer(smoothed_mus[0], smoothed_mus[0])) if self.model.emission_distn.affine: ExxT = blockarray([[ExxT,np.atleast_2d(Ex).T], [np.atleast_2d(Ex),np.atleast_2d(self.T)]]) if not obs_time.size > 1: ExxTe = ExxT return ExxT, E_xt_xtT, E_xtp1_xtp1T, ExxTe
def log_likelihood(self, xy): assert isinstance(xy, (tuple, np.ndarray)) A, sigma, D = self.A, self.sigma, self.D_out x, y = (xy[:, :-D], xy[:, -D:]) if isinstance(xy, np.ndarray) else xy if self.affine: A, b = A[:, :-1], A[:, -1] sigma_inv, L = inv_psd(sigma, return_chol=True) parammat = -1. / 2 * blockarray([[ A.T.dot(sigma_inv).dot(A), -A.T.dot(sigma_inv) ], [-sigma_inv.dot(A), sigma_inv]]) contract = 'ni,ni->n' if x.ndim == 2 else 'i,i->' if isinstance(xy, np.ndarray): out = np.einsum(contract, xy.dot(parammat), xy) else: out = np.einsum(contract, x.dot(parammat[:-D, :-D]), x) out += np.einsum(contract, y.dot(parammat[-D:, -D:]), y) out += 2 * np.einsum(contract, x.dot(parammat[:-D, -D:]), y) out -= D / 2 * np.log(2 * np.pi) + np.log(np.diag(L)).sum() if self.affine: out += y.dot(sigma_inv).dot(b) out -= x.dot(A.T).dot(sigma_inv).dot(b) out -= 1. / 2 * b.dot(sigma_inv).dot(b) return out
def log_likelihood(self,xy): assert isinstance(xy, (tuple,np.ndarray)) A, sigma, D = self.A, self.sigma, self.D_out x, y = (xy[:,:-D], xy[:,-D:]) if isinstance(xy,np.ndarray) else xy if self.affine: A, b = A[:,:-1], A[:,-1] sigma_inv, L = inv_psd(sigma, return_chol=True) parammat = -1./2 * blockarray([ [A.T.dot(sigma_inv).dot(A), -A.T.dot(sigma_inv)], [-sigma_inv.dot(A), sigma_inv]]) contract = 'ni,ni->n' if x.ndim == 2 else 'i,i->' if isinstance(xy, np.ndarray): out = np.einsum(contract,xy.dot(parammat),xy) else: out = np.einsum(contract,x.dot(parammat[:-D,:-D]),x) out += np.einsum(contract,y.dot(parammat[-D:,-D:]),y) out += 2*np.einsum(contract,x.dot(parammat[:-D,-D:]),y) out -= D/2*np.log(2*np.pi) + np.log(np.diag(L)).sum() if self.affine: out += y.dot(sigma_inv).dot(b) out -= x.dot(A.T).dot(sigma_inv).dot(b) out -= 1./2*b.dot(sigma_inv).dot(b) return out
def _param_matrix(o): D, A, sigma = o.D, o.A, o.sigma sigma_inv = np.linalg.inv(sigma) parammat = -1./2 * blockarray([ [A.T.dot(sigma_inv).dot(A), -A.T.dot(sigma_inv)], [-sigma_inv.dot(A), sigma_inv] ]) normalizer = D/2*np.log(2*np.pi) + np.log(np.diag(np.linalg.cholesky(sigma))).sum() return parammat, normalizer
def _param_matrix(o): D, A, sigma = o.D, o.A, o.sigma sigma_inv = np.linalg.inv(sigma) parammat = -1. / 2 * blockarray([[ A.T.dot(sigma_inv).dot(A), -A.T.dot(sigma_inv) ], [-sigma_inv.dot(A), sigma_inv]]) normalizer = D / 2 * np.log(2 * np.pi) + np.log( np.diag(np.linalg.cholesky(sigma))).sum() return parammat, normalizer
def expected_log_likelihood(self, xy=None, stats=None): # TODO test values, test for the affine case assert isinstance(xy, (tuple, np.ndarray)) ^ isinstance(stats, tuple) D = self.D_out E_Sigmainv, E_Sigmainv_A, E_AT_Sigmainv_A, E_logdetSigmainv = \ mniw_expectedstats( *self._natural_to_standard(self.mf_natural_hypparam)) if self.affine: E_Sigmainv_A, E_Sigmainv_b = \ E_Sigmainv_A[:,:-1], E_Sigmainv_A[:,-1] E_AT_Sigmainv_A, E_AT_Sigmainv_b, E_bT_Sigmainv_b = \ E_AT_Sigmainv_A[:-1,:-1], E_AT_Sigmainv_A[:-1,-1], \ E_AT_Sigmainv_A[-1,-1] if xy is not None: x, y = (xy[:,:-D], xy[:,-D:]) if isinstance(xy, np.ndarray) \ else xy parammat = -1./2 * blockarray([ [E_AT_Sigmainv_A, -E_Sigmainv_A.T], [-E_Sigmainv_A, E_Sigmainv]]) contract = 'ni,ni->n' if x.ndim == 2 else 'i,i->' if isinstance(xy, np.ndarray): out = np.einsum('ni,ni->n', xy.dot(parammat), xy) else: out = np.einsum(contract,x.dot(parammat[:-D,:-D]),x) out += np.einsum(contract,y.dot(parammat[-D:,-D:]),y) out += 2*np.einsum(contract,x.dot(parammat[:-D,-D:]),y) out += -D/2*np.log(2*np.pi) + 1./2*E_logdetSigmainv if self.affine: out += y.dot(E_Sigmainv_b) out -= x.dot(E_AT_Sigmainv_b) out -= 1./2 * E_bT_Sigmainv_b else: if self.affine: Ey, Ex = stats[:2] yyT, yxT, xxT, n = stats[-4:] contract = 'ij,nij->n' if yyT.ndim == 3 else 'ij,ij->' out = -1./2 * np.einsum(contract, E_AT_Sigmainv_A, xxT) out += np.einsum(contract, E_Sigmainv_A, yxT) out += -1./2 * np.einsum(contract, E_Sigmainv, yyT) out += -D/2*np.log(2*np.pi) + n/2.*E_logdetSigmainv if self.affine: out += Ey.dot(E_Sigmainv_b) out -= Ex.dot(E_AT_Sigmainv_b) out -= 1./2 * E_bT_Sigmainv_b return out
def expected_log_likelihood(self, xy=None, stats=None): # TODO test values, test for the affine case assert isinstance(xy, (tuple, np.ndarray)) ^ isinstance(stats, tuple) D = self.D_out E_Sigmainv, E_Sigmainv_A, E_AT_Sigmainv_A, E_logdetSigmainv = \ mniw_expectedstats( *self._natural_to_standard(self.mf_natural_hypparam)) if self.affine: E_Sigmainv_A, E_Sigmainv_b = \ E_Sigmainv_A[:,:-1], E_Sigmainv_A[:,-1] E_AT_Sigmainv_A, E_AT_Sigmainv_b, E_bT_Sigmainv_b = \ E_AT_Sigmainv_A[:-1,:-1], E_AT_Sigmainv_A[:-1,-1], \ E_AT_Sigmainv_A[-1,-1] if xy is not None: x, y = (xy[:,:-D], xy[:,-D:]) if isinstance(xy, np.ndarray) \ else xy parammat = -1. / 2 * blockarray([[ E_AT_Sigmainv_A, -E_Sigmainv_A.T ], [-E_Sigmainv_A, E_Sigmainv]]) contract = 'ni,ni->n' if x.ndim == 2 else 'i,i->' if isinstance(xy, np.ndarray): out = np.einsum('ni,ni->n', xy.dot(parammat), xy) else: out = np.einsum(contract, x.dot(parammat[:-D, :-D]), x) out += np.einsum(contract, y.dot(parammat[-D:, -D:]), y) out += 2 * np.einsum(contract, x.dot(parammat[:-D, -D:]), y) out += -D / 2 * np.log(2 * np.pi) + 1. / 2 * E_logdetSigmainv if self.affine: out += y.dot(E_Sigmainv_b) out -= x.dot(E_AT_Sigmainv_b) out -= 1. / 2 * E_bT_Sigmainv_b else: if self.affine: Ey, Ex = stats[:2] yyT, yxT, xxT, n = stats[-4:] contract = 'ij,nij->n' if yyT.ndim == 3 else 'ij,ij->' out = -1. / 2 * np.einsum(contract, E_AT_Sigmainv_A, xxT) out += np.einsum(contract, E_Sigmainv_A, yxT) out += -1. / 2 * np.einsum(contract, E_Sigmainv, yyT) out += -D / 2 * np.log(2 * np.pi) + n / 2. * E_logdetSigmainv if self.affine: out += Ey.dot(E_Sigmainv_b) out -= Ex.dot(E_AT_Sigmainv_b) out -= 1. / 2 * E_bT_Sigmainv_b return out
def _get_scaled_statistics(self, data, precisions): assert isinstance(data, (list, tuple, np.ndarray)) if isinstance(data, list): return sum((self._get_scaled_statistics(d, p) for d, p in zip(data, precisions)), self._empty_statistics()) elif isinstance(data, tuple): x, y = data bad = np.isnan(x).any(1) | np.isnan(y).any(1) x, y = x[~bad], y[~bad] precisions = precisions[~bad] sqrt_prec = np.sqrt(precisions) n, D = y.shape if self.affine: x = np.column_stack((x, np.ones(n))) # Scale by the precision # xs = x * sqrt_prec[:, na] # ys = y * sqrt_prec[:, na] xs = x * np.tile(sqrt_prec[:, None], (1, x.shape[1])) ys = y * np.tile(sqrt_prec[:, None], (1, D)) xxT, yxT, yyT = xs.T.dot(xs), ys.T.dot(xs), ys.T.dot(ys) return np.array([yyT, yxT, xxT, n]) else: # data passed in like np.hstack((x, y)) # x, y = data[:,:-self.D_out], data[:,-self.D_out:] # return self._get_scaled_statistics((x, y), precisions) bad = np.isnan(data).any(1) data = data[~bad] precisions = precisions[~bad] n, D = data.shape[0], self.D_out # This tile call is suboptimal but without it we can hit issues # with strided data, as in autoregressive models. scaled_data = data * np.tile(precisions[:, None], (1, data.shape[1])) statmat = scaled_data.T.dot(data) xxT, yxT, yyT = \ statmat[:-D,:-D], statmat[-D:,:-D], statmat[-D:,-D:] if self.affine: xy = scaled_data.sum(0) x, y = xy[:-D], xy[-D:] xxT = blockarray([[xxT, x[:, na]], [x[na, :], np.atleast_2d(precisions.sum())]]) yxT = np.hstack((yxT, y[:, na])) return np.array([yyT, yxT, xxT, n])
def _get_weighted_statistics(self, data, weights): assert isinstance(data, (list, tuple, np.ndarray)) if isinstance(data, list): return sum((self._get_statistics(d) for d in data), self._empty_statistics()) elif isinstance(data, tuple): x, y = data bad = np.isnan(x).any(1) | np.isnan(y).any(1) x, y, weights = x[~bad], y[~bad], weights[~bad] n, D = weights.sum(), y.shape[1] wx = weights[:, na] * x xxT, yxT, yyT = \ x.T.dot(wx), y.T.dot(wx), y.T.dot(weights[:,na]*y) if self.affine: x, y = weights.dot(x), weights.dot(y) xxT = blockarray([[xxT, x[:, na]], [x[na, :], np.atleast_2d(n)]]) yxT = np.hstack((yxT, y[:, na])) return np.array([yyT, yxT, xxT, n]) else: # data passed in like np.hstack((x, y)) gi = ~np.isnan(data).any(1) data, weights = data[gi], weights[gi] n, D = weights.sum(), self.D_out statmat = data.T.dot(weights[:, na] * data) xxT, yxT, yyT = \ statmat[:-D,:-D], statmat[-D:,:-D], statmat[-D:,-D:] if self.affine: xy = weights.dot(data) x, y = xy[:-D], xy[-D:] xxT = blockarray([[xxT, x[:, na]], [x[na, :], np.atleast_2d(n)]]) yxT = np.hstack((yxT, y[:, na])) return np.array([yyT, yxT, xxT, n])
def _stats_ensure_array(stats): if isinstance(stats, np.ndarray): return stats affine = len(stats) > 4 yyT, yxT, xxT, n = stats[-4:] if affine: y, x = stats[:2] yxT = np.hstack((yxT, y[:, None])) xxT = blockarray([[xxT, x[:, None]], [x[None, :], 1.]]) return np.array([yyT, yxT, xxT, n])
def _stats_ensure_array(stats): if isinstance(stats, np.ndarray): return stats affine = len(stats) > 4 yyT, yxT, xxT, n = stats[-4:] if affine: y, x = stats[:2] yxT = np.hstack((yxT, y[:,None])) xxT = blockarray([[xxT, x[:,None]], [x[None,:], 1.]]) return np.array([yyT, yxT, xxT, n])
def _get_weighted_statistics(self,data,weights): assert isinstance(data, (list, tuple, np.ndarray)) if isinstance(data,list): return sum((self._get_statistics(d) for d in data), self._empty_statistics()) elif isinstance(data, tuple): x, y = data bad = np.isnan(x).any(1) | np.isnan(y).any(1) x, y, weights = x[~bad], y[~bad], weights[~bad] n, D = weights.sum(), y.shape[1] wx = weights[:,na]*x xxT, yxT, yyT = \ x.T.dot(wx), y.T.dot(wx), y.T.dot(weights[:,na]*y) if self.affine: x, y = weights.dot(x), weights.dot(y) xxT = blockarray([[xxT,x[:,na]],[x[na,:],np.atleast_2d(n)]]) yxT = np.hstack((yxT,y[:,na])) return np.array([yyT, yxT, xxT, n]) else: # data passed in like np.hstack((x, y)) gi = ~np.isnan(data).any(1) data, weights = data[gi], weights[gi] n, D = weights.sum(), self.D_out statmat = data.T.dot(weights[:,na]*data) xxT, yxT, yyT = \ statmat[:-D,:-D], statmat[-D:,:-D], statmat[-D:,-D:] if self.affine: xy = weights.dot(data) x, y = xy[:-D], xy[-D:] xxT = blockarray([[xxT,x[:,na]],[x[na,:],np.atleast_2d(n)]]) yxT = np.hstack((yxT,y[:,na])) return np.array([yyT, yxT, xxT, n])
def _get_scaled_statistics(self, data, precisions): assert isinstance(data, (list, tuple, np.ndarray)) if isinstance(data,list): return sum((self._get_scaled_statistics(d, p) for d, p in zip(data, precisions)), self._empty_statistics()) elif isinstance(data, tuple): x, y = data bad = np.isnan(x).any(1) | np.isnan(y).any(1) x, y = x[~bad], y[~bad] precisions = precisions[~bad] sqrt_prec = np.sqrt(precisions) n, D = y.shape if self.affine: x = np.column_stack((x, np.ones(n))) # Scale by the precision # xs = x * sqrt_prec[:, na] # ys = y * sqrt_prec[:, na] xs = x * np.tile(sqrt_prec[:, None], (1, x.shape[1])) ys = y * np.tile(sqrt_prec[:, None], (1, D)) xxT, yxT, yyT = xs.T.dot(xs), ys.T.dot(xs), ys.T.dot(ys) return np.array([yyT, yxT, xxT, n]) else: # data passed in like np.hstack((x, y)) # x, y = data[:,:-self.D_out], data[:,-self.D_out:] # return self._get_scaled_statistics((x, y), precisions) bad = np.isnan(data).any(1) data = data[~bad] precisions = precisions[~bad] n, D = data.shape[0], self.D_out # This tile call is suboptimal but without it we can hit issues # with strided data, as in autoregressive models. scaled_data = data * np.tile(precisions[:,None], (1, data.shape[1])) statmat = scaled_data.T.dot(data) xxT, yxT, yyT = \ statmat[:-D,:-D], statmat[-D:,:-D], statmat[-D:,-D:] if self.affine: xy = scaled_data.sum(0) x, y = xy[:-D], xy[-D:] xxT = blockarray([[xxT, x[:,na]], [x[na,:], np.atleast_2d(precisions.sum())]]) yxT = np.hstack((yxT, y[:,na])) return np.array([yyT, yxT, xxT, n])
def _get_statistics(self,data): if isinstance(data,list): return sum((self._get_statistics(d) for d in data), self._empty_statistics()) else: data = data[~np.isnan(data).any(1)] n, D = data.shape[0], self.D_out statmat = data.T.dot(data) xxT, yxT, yyT = \ statmat[:-D,:-D], statmat[-D:,:-D], statmat[-D:,-D:] if self.affine: xy = data.sum(0) x, y = xy[:-D], xy[-D:] xxT = blockarray([[xxT,x[:,na]],[x[na,:],np.atleast_2d(n)]]) yxT = np.hstack((yxT,y[:,na])) return np.array([yyT, yxT, xxT, n])
def _get_weighted_statistics(self,data,weights): if isinstance(data,list): return sum((self._get_statistics(d) for d in data), self._empty_statistics()) else: gi = ~np.isnan(data).any(1) data, weights = data[gi], weights[gi] n, D = weights.sum(), self.D_out statmat = data.T.dot(weights[:,na]*data) xxT, yxT, yyT = \ statmat[:-D,:-D], statmat[-D:,:-D], statmat[-D:,-D:] if self.affine: xy = weights.dot(data) x, y = xy[:-D], xy[-D:] xxT = blockarray([[xxT,x[:,na]],[x[na,:],np.atleast_2d(n)]]) yxT = np.hstack((yxT,y[:,na])) return np.array([yyT, yxT, xxT, n])
def log_likelihood(self,xy): A, sigma, D = self.A, self.sigma, self.D_out x, y = xy[:,:-D], xy[:,-D:] if self.affine: A, b = A[:,:-1], A[:,-1] sigma_inv, L = inv_psd(sigma, return_chol=True) parammat = -1./2 * blockarray([ [A.T.dot(sigma_inv).dot(A), -A.T.dot(sigma_inv)], [-sigma_inv.dot(A), sigma_inv] ]) out = np.einsum('ni,ni->n',xy.dot(parammat),xy) out -= D/2*np.log(2*np.pi) + np.log(np.diag(L)).sum() if self.affine: out += y.dot(sigma_inv).dot(b) out -= x.dot(A.T).dot(sigma_inv).dot(b) out -= 1./2*b.dot(sigma_inv).dot(b) return out