def __init__(self, K, n, dynamics_distn, init_dynamics_distn, C=None, sigma_C=1.0, mu_pi=None): self.K = K self.n = n self.init_dynamics_distn = init_dynamics_distn self.mu_init = self.init_dynamics_distn.mu self.sigma_init = self.init_dynamics_distn.sigma self.dynamics_distn = dynamics_distn if mu_pi is None: mu_pi = np.ones(self.K) / self.K self.emission_distn = PGMultinomialRegression(K, n, C=C, sigma_C=sigma_C, mu_pi=mu_pi) # Initialize a list of augmented data dicts self.data_list = []
def __init__( self, K, n, dynamics_distn, init_dynamics_distn, C=None, sigma_C=1.0, mu_pi=None): self.K = K self.n = n self.init_dynamics_distn = init_dynamics_distn self.mu_init = self.init_dynamics_distn.mu self.sigma_init = self.init_dynamics_distn.sigma self.dynamics_distn = dynamics_distn if mu_pi is None: mu_pi = np.ones(self.K)/self.K self.emission_distn = PGMultinomialRegression( K, n, C=C, sigma_C=sigma_C, mu_pi=mu_pi) # Initialize a list of augmented data dicts self.data_list = []
class _MultinomialLDSBase(Model): def __init__(self, K, n, dynamics_distn, init_dynamics_distn, C=None, sigma_C=1.0, mu_pi=None): self.K = K self.n = n self.init_dynamics_distn = init_dynamics_distn self.mu_init = self.init_dynamics_distn.mu self.sigma_init = self.init_dynamics_distn.sigma self.dynamics_distn = dynamics_distn if mu_pi is None: mu_pi = np.ones(self.K) / self.K self.emission_distn = PGMultinomialRegression(K, n, C=C, sigma_C=sigma_C, mu_pi=mu_pi) # Initialize a list of augmented data dicts self.data_list = [] @property def p(self): return self.K - 1 @property def A(self): return self.dynamics_distn.A @A.setter def A(self, A): self.dynamics_distn.A = A @property def B(self): # TODO: Allow input dependence return np.zeros((self.n, 0)) @property def sigma_states(self): return self.dynamics_distn.sigma @sigma_states.setter def sigma_states(self, sigma_states): self.dynamics_distn.sigma = sigma_states @property def C(self): return self.emission_distn.C @C.setter def C(self, C): self.emission_distn.C = C @property def D(self): # TODO: Allow input dependence return np.zeros((self.p, 0)) @property def mu_init(self): return self.init_dynamics_distn.mu @mu_init.setter def mu_init(self, mu): self.init_dynamics_distn.mu = mu @property def sigma_init(self): return self.init_dynamics_distn.sigma @sigma_init.setter def sigma_init(self, sigma): self.init_dynamics_distn.sigma = sigma def add_data(self, data): assert isinstance(data, np.ndarray) \ and data.ndim == 2 and data.shape[1] == self.K T = data.shape[0] augmented_data = {"x": data, "T": T} augmented_data["states"] = MultinomialLDSStates(model=self, data=data) self.emission_distn.augment_data(augmented_data) self.data_list.append(augmented_data) def log_likelihood(self): return sum( self.emission_distn.log_likelihood(data) for data in self.data_list) def heldout_log_likelihood(self, X, M=100): return self._mc_heldout_log_likelihood(X, M) def _info_form_heldout_log_likelihood(self, X, M=10): """ We can analytically integrate out z (latent states) given omega. To estimate the heldout log likelihood of a data sequence, we Monte Carlo integrate over omega, where omega is drawn from the prior. :param data: :param M: number of Monte Carlo samples for integrating out omega :return: """ # assert len(self.data_list) == 1, "TODO: Support more than 1 data set" T, K = X.shape assert K == self.K kappa = kappa_vec(X) N = N_vec(X) # Compute the data-specific normalization constant from the # augmented multinomial distribution Z_mul = (gammaln(N + 1) - gammaln(X[:, :-1] + 1) - gammaln(N - X[:, :-1] + 1)).sum() Z_mul += (-N * np.log(2.)).sum() # Monte carlo integrate wrt omega ~ PG(N, 0) import pypolyagamma as ppg hlls = np.zeros(M) for m in range(M): # Sample omega using the emission distributions samplers omega = np.zeros(N.size) ppg.pgdrawvpar(self.emission_distn.ppgs, N.ravel(), np.zeros(N.size), omega) omega = omega.reshape((T, K - 1)) # Exactly integrate out the latent states z using message passing # The "data" is the normal potential from the states = MultinomialLDSStates(model=self, data=X) conditional_mean = kappa / np.clip( omega, 1e-64, np.inf) - self.emission_distn.mu[None, :] conditional_prec = np.zeros((T, K - 1, K - 1)) for t in range(T): conditional_prec[t, :, :] = np.diag(omega[t, :]) Z_lds = states.info_log_likelihood(conditional_mean, conditional_prec) # Sum them up to get the heldout log likelihood for this omega hlls[m] = Z_mul + Z_lds # Now take the log of the average to get the log likelihood hll = logsumexp(hlls) - np.log(M) # Use bootstrap to compute error bars samples = np.random.choice(hlls, size=(100, M), replace=True) hll_samples = logsumexp(samples, axis=1) - np.log(M) std_hll = hll_samples.std() return hll, std_hll def _distn_form_heldout_log_likelihood(self, X, M=10): """ We can analytically integrate out z (latent states) given omega. To estimate the heldout log likelihood of a data sequence, we Monte Carlo integrate over omega, where omega is drawn from the prior. :param data: :param M: number of Monte Carlo samples for integrating out omega :return: """ # assert len(self.data_list) == 1, "TODO: Support more than 1 data set" T, K = X.shape assert K == self.K kappa = kappa_vec(X) N = N_vec(X) # Compute the data-specific normalization constant from the # augmented multinomial distribution Z_mul = (gammaln(N + 1) - gammaln(X[:, :-1] + 1) - gammaln(N - X[:, :-1] + 1)).sum() Z_mul += (-N * np.log(2.)).sum() # Monte carlo integrate wrt omega ~ PG(N, 0) import pypolyagamma as ppg hlls = np.zeros(M) for m in range(M): # Sample omega using the emission distributions samplers omega = np.zeros(N.size) ppg.pgdrawvpar(self.emission_distn.ppgs, N.ravel(), np.zeros(N.size), omega) omega = omega.reshape((T, K - 1)) # valid = omega > 0 omega = np.clip(omega, 1e-8, np.inf) # Compute the normalization constant for this omega Z_omg = 0.5 * (kappa**2 / omega).sum() Z_omg += T * (K - 1) / 2. * np.log(2 * np.pi) Z_omg += -0.5 * np.log(omega).sum() # 1/2 log det of Omega_t^{-1} # clip omega = zero for message passing # omega[~valid] = 1e-32 # Exactly integrate out the latent states z using message passing # The "data" is the normal potential from the states = MultinomialLDSStates(model=self, data=X) conditional_mean = kappa / omega - self.emission_distn.mu[None, :] # conditional_mean[~np.isfinite(conditional_mean)] = 0 conditional_cov = np.zeros((T, K - 1, K - 1)) for t in range(T): conditional_cov[t, :, :] = np.diag(1. / omega[t, :]) Z_lds = states.log_likelihood(conditional_mean, conditional_cov) # Sum them up to get the heldout log likelihood for this omega hlls[m] = Z_mul + Z_omg + Z_lds # Now take the log of the average to get the log likelihood hll = logsumexp(hlls) - np.log(M) # Use bootstrap to compute error bars samples = np.random.choice(hlls, size=(100, M), replace=True) hll_samples = logsumexp(samples, axis=1) - np.log(M) std_hll = hll_samples.std() return hll, std_hll def _mc_heldout_log_likelihood(self, X, M=100): # Estimate the held out likelihood using Monte Carlo T, K = X.shape assert K == self.K lls = np.zeros(M) for m in range(M): # Sample latent states from the prior data = self.generate(T=T, keep=False) data["x"] = X lls[m] = self.emission_distn.log_likelihood(data) # Compute the average hll = logsumexp(lls) - np.log(M) # Use bootstrap to compute error bars samples = np.random.choice(lls, size=(100, M), replace=True) hll_samples = logsumexp(samples, axis=1) - np.log(M) std_hll = hll_samples.std() return hll, std_hll def predictive_log_likelihood(self, X_pred, data_index=0, M=100): """ Hacky way of computing the predictive log likelihood :param X_pred: :param data_index: :param M: :return: """ Tpred = X_pred.shape[0] data = self.data_list[data_index] conditional_mean = self.emission_distn.conditional_mean(data) conditional_cov = self.emission_distn.conditional_cov(data, flat=True) lls = [] z_preds = data["states"].predict_states(conditional_mean, conditional_cov, Tpred=Tpred, Npred=M) for m in range(M): ll_pred = self.emission_distn.log_likelihood({ "z": z_preds[..., m], "x": X_pred }) lls.append(ll_pred) # Compute the average hll = logsumexp(lls) - np.log(M) # Use bootstrap to compute error bars samples = np.random.choice(lls, size=(100, M), replace=True) hll_samples = logsumexp(samples, axis=1) - np.log(M) std_hll = hll_samples.std() return hll, std_hll def pi(self, data): return self.emission_distn.pi(data) def psi(self, data): return self.emission_distn.psi(data) def generate(self, T=100, N=1, keep=True): augmented_data = {"T": T} # Sample latent state sequence states = MultinomialLDSStates(model=self, T=T, initialize_from_prior=True) augmented_data["states"] = states # Sample observations x = self.emission_distn.rvs(z=states.stateseq, N=N) states.data = x augmented_data["x"] = x if keep: self.data_list.append(augmented_data) return augmented_data def rvs(self, T): return self.generate(keep=False, T=T)[0]
class _MultinomialLDSBase(Model): def __init__( self, K, n, dynamics_distn, init_dynamics_distn, C=None, sigma_C=1.0, mu_pi=None): self.K = K self.n = n self.init_dynamics_distn = init_dynamics_distn self.mu_init = self.init_dynamics_distn.mu self.sigma_init = self.init_dynamics_distn.sigma self.dynamics_distn = dynamics_distn if mu_pi is None: mu_pi = np.ones(self.K)/self.K self.emission_distn = PGMultinomialRegression( K, n, C=C, sigma_C=sigma_C, mu_pi=mu_pi) # Initialize a list of augmented data dicts self.data_list = [] @property def p(self): return self.K-1 @property def A(self): return self.dynamics_distn.A @A.setter def A(self, A): self.dynamics_distn.A = A @property def B(self): # TODO: Allow input dependence return np.zeros((self.n, 0)) @property def sigma_states(self): return self.dynamics_distn.sigma @sigma_states.setter def sigma_states(self, sigma_states): self.dynamics_distn.sigma = sigma_states @property def C(self): return self.emission_distn.C @C.setter def C(self,C): self.emission_distn.C = C @property def D(self): # TODO: Allow input dependence return np.zeros((self.p, 0)) @property def mu_init(self): return self.init_dynamics_distn.mu @mu_init.setter def mu_init(self, mu): self.init_dynamics_distn.mu = mu @property def sigma_init(self): return self.init_dynamics_distn.sigma @sigma_init.setter def sigma_init(self, sigma): self.init_dynamics_distn.sigma = sigma def add_data(self, data): assert isinstance(data, np.ndarray) \ and data.ndim == 2 and data.shape[1] == self.K T = data.shape[0] augmented_data = {"x": data, "T": T} augmented_data["states"] = MultinomialLDSStates(model=self, data=data) self.emission_distn.augment_data(augmented_data) self.data_list.append(augmented_data) def log_likelihood(self): return sum(self.emission_distn.log_likelihood(data) for data in self.data_list) def heldout_log_likelihood(self, X, M=100): return self._mc_heldout_log_likelihood(X, M) def _info_form_heldout_log_likelihood(self, X, M=10): """ We can analytically integrate out z (latent states) given omega. To estimate the heldout log likelihood of a data sequence, we Monte Carlo integrate over omega, where omega is drawn from the prior. :param data: :param M: number of Monte Carlo samples for integrating out omega :return: """ # assert len(self.data_list) == 1, "TODO: Support more than 1 data set" T, K = X.shape assert K == self.K kappa = kappa_vec(X) N = N_vec(X) # Compute the data-specific normalization constant from the # augmented multinomial distribution Z_mul = (gammaln(N + 1) - gammaln(X[:,:-1]+1) - gammaln(N-X[:,:-1]+1)).sum() Z_mul += (-N * np.log(2.)).sum() # Monte carlo integrate wrt omega ~ PG(N, 0) import pypolyagamma as ppg hlls = np.zeros(M) for m in range(M): # Sample omega using the emission distributions samplers omega = np.zeros(N.size) ppg.pgdrawvpar(self.emission_distn.ppgs, N.ravel(), np.zeros(N.size), omega) omega = omega.reshape((T, K-1)) # Exactly integrate out the latent states z using message passing # The "data" is the normal potential from the states = MultinomialLDSStates(model=self, data=X) conditional_mean = kappa / np.clip(omega, 1e-64,np.inf) - self.emission_distn.mu[None, :] conditional_prec = np.zeros((T, K-1, K-1)) for t in range(T): conditional_prec[t,:,:] = np.diag(omega[t,:]) Z_lds = states.info_log_likelihood(conditional_mean, conditional_prec) # Sum them up to get the heldout log likelihood for this omega hlls[m] = Z_mul + Z_lds # Now take the log of the average to get the log likelihood hll = logsumexp(hlls) - np.log(M) # Use bootstrap to compute error bars samples = np.random.choice(hlls, size=(100, M), replace=True) hll_samples = logsumexp(samples, axis=1) - np.log(M) std_hll = hll_samples.std() return hll, std_hll def _distn_form_heldout_log_likelihood(self, X, M=10): """ We can analytically integrate out z (latent states) given omega. To estimate the heldout log likelihood of a data sequence, we Monte Carlo integrate over omega, where omega is drawn from the prior. :param data: :param M: number of Monte Carlo samples for integrating out omega :return: """ # assert len(self.data_list) == 1, "TODO: Support more than 1 data set" T, K = X.shape assert K == self.K kappa = kappa_vec(X) N = N_vec(X) # Compute the data-specific normalization constant from the # augmented multinomial distribution Z_mul = (gammaln(N + 1) - gammaln(X[:,:-1]+1) - gammaln(N-X[:,:-1]+1)).sum() Z_mul += (-N * np.log(2.)).sum() # Monte carlo integrate wrt omega ~ PG(N, 0) import pypolyagamma as ppg hlls = np.zeros(M) for m in range(M): # Sample omega using the emission distributions samplers omega = np.zeros(N.size) ppg.pgdrawvpar(self.emission_distn.ppgs, N.ravel(), np.zeros(N.size), omega) omega = omega.reshape((T, K-1)) # valid = omega > 0 omega = np.clip(omega, 1e-8, np.inf) # Compute the normalization constant for this omega Z_omg = 0.5 * (kappa**2/omega).sum() Z_omg += T * (K-1)/2. * np.log(2*np.pi) Z_omg += -0.5 * np.log(omega).sum() # 1/2 log det of Omega_t^{-1} # clip omega = zero for message passing # omega[~valid] = 1e-32 # Exactly integrate out the latent states z using message passing # The "data" is the normal potential from the states = MultinomialLDSStates(model=self, data=X) conditional_mean = kappa / omega - self.emission_distn.mu[None, :] # conditional_mean[~np.isfinite(conditional_mean)] = 0 conditional_cov = np.zeros((T, K-1, K-1)) for t in range(T): conditional_cov[t,:,:] = np.diag(1./omega[t,:]) Z_lds = states.log_likelihood(conditional_mean, conditional_cov) # Sum them up to get the heldout log likelihood for this omega hlls[m] = Z_mul + Z_omg + Z_lds # Now take the log of the average to get the log likelihood hll = logsumexp(hlls) - np.log(M) # Use bootstrap to compute error bars samples = np.random.choice(hlls, size=(100, M), replace=True) hll_samples = logsumexp(samples, axis=1) - np.log(M) std_hll = hll_samples.std() return hll, std_hll def _mc_heldout_log_likelihood(self, X, M=100): # Estimate the held out likelihood using Monte Carlo T, K = X.shape assert K == self.K lls = np.zeros(M) for m in range(M): # Sample latent states from the prior data = self.generate(T=T, keep=False) data["x"] = X lls[m] = self.emission_distn.log_likelihood(data) # Compute the average hll = logsumexp(lls) - np.log(M) # Use bootstrap to compute error bars samples = np.random.choice(lls, size=(100, M), replace=True) hll_samples = logsumexp(samples, axis=1) - np.log(M) std_hll = hll_samples.std() return hll, std_hll def predictive_log_likelihood(self, X_pred, data_index=0, M=100): """ Hacky way of computing the predictive log likelihood :param X_pred: :param data_index: :param M: :return: """ Tpred = X_pred.shape[0] data = self.data_list[data_index] conditional_mean = self.emission_distn.conditional_mean(data) conditional_cov = self.emission_distn.conditional_cov(data, flat=True) lls = [] z_preds = data["states"].predict_states( conditional_mean, conditional_cov, Tpred=Tpred, Npred=M) for m in range(M): ll_pred = self.emission_distn.log_likelihood( {"z": z_preds[...,m], "x": X_pred}) lls.append(ll_pred) # Compute the average hll = logsumexp(lls) - np.log(M) # Use bootstrap to compute error bars samples = np.random.choice(lls, size=(100, M), replace=True) hll_samples = logsumexp(samples, axis=1) - np.log(M) std_hll = hll_samples.std() return hll, std_hll def pi(self, data): return self.emission_distn.pi(data) def psi(self, data): return self.emission_distn.psi(data) def generate(self, T=100, N=1, keep=True): augmented_data = {"T": T} # Sample latent state sequence states = MultinomialLDSStates(model=self, T=T, initialize_from_prior=True) augmented_data["states"] = states # Sample observations x = self.emission_distn.rvs(z=states.stateseq, N=N) states.data = x augmented_data["x"] = x if keep: self.data_list.append(augmented_data) return augmented_data def rvs(self,T): return self.generate(keep=False, T=T)[0]