def svi_example(true_model, true_data): X, mask = true_data.X, true_data.mask # Fit a test model model = FactorAnalysis( D_obs, D_latent, # W=true_model.W, sigmasq=true_model.sigmasq ) # Add the data in minibatches minibatchsize = 250 for start in range(0, N, minibatchsize): end = min(start + minibatchsize, N) model.add_data(X[start:end], mask=mask[start:end]) lps = [] angles = [] N_iters = 100 delay = 10.0 forgetting_rate = 0.75 stepsize = (np.arange(N_iters) + delay)**(-forgetting_rate) for itr in progprint_xrange(N_iters): lps.append(model.meanfield_sgdstep(stepsize[itr])) E_W, _, _, _ = model.regression.mf_expectations angles.append(principal_angle(true_model.W, E_W)) Z_inf = model.data_list[0].E_Z Z_true = true_data.Z[:Z_inf.shape[0]] plot_results(lps, angles, Z_true, Z_inf)
def fit_hmm(Xs, Xtest, D_hmm, N_samples=100): print("Fitting HMM with %d states" % D_hmm) model = MultinomialHMM(K, D_hmm, alpha_0=10.0) for X in Xs: model.add_data(X) compute_pred_ll = lambda: sum([model.log_likelihood(np.vstack((Xs[i], Xtest[i]))) - model.log_likelihood(Xs[i]) for i,Xt in enumerate(Xtest)]) init_results = (0, None, model.log_likelihood(), model.log_likelihood(Xtest), compute_pred_ll()) def resample(): tic = time.time() model.resample_model() toc = time.time() - tic return toc, None, model.log_likelihood(), \ np.nan, \ compute_pred_ll() times, samples, lls, test_lls, pred_lls = \ map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples, perline=5)]))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
def svi_example(true_model, X, Z_true, mask): # Fit a test model model = FactorAnalysis( D_obs, D_latent, # W=true_model.W, sigmasq=true_model.sigmasq ) # Add the data in minibatches N = X.shape[0] minibatchsize = 200 prob = minibatchsize / float(N) lps = [] angles = [] N_iters = 100 delay = 10.0 forgetting_rate = 0.75 stepsize = (np.arange(N_iters) + delay)**(-forgetting_rate) for itr in progprint_xrange(N_iters): minibatch = np.random.permutation(N)[:minibatchsize] X_mb, mask_mb = X[minibatch], mask[minibatch] lps.append(model.meanfield_sgdstep(X_mb, prob, stepsize[itr], masks=mask_mb)) E_W, _, _, _ = model.regression.mf_expectations angles.append(principal_angle(true_model.W, E_W)) # Compute the expected states for the first minibatch of data model.add_data(X, mask) statesobj = model.data_list.pop() statesobj.meanfieldupdate() Z_inf = statesobj.E_Z plot_results(lps, angles, Z_true, Z_inf)
def fit(name, model, test_data, N_iter=1000, init_state_seq=None): def evaluate(model): ll = model.log_likelihood() pll = model.log_likelihood(test_data) N_used = len(model.used_states) trans = model.trans_distn alpha = trans.alpha gamma = trans.gamma if hasattr(trans, "gamma") else None rates = model.rates.copy() obs_hypers = model.obs_hypers # print 'N_states: {}, \tPLL:{}\n'.format(len(model.used_states), pll), return ll, pll, N_used, alpha, gamma, rates, obs_hypers def sample(model): tic = time.time() model.resample_model() timestep = time.time() - tic return evaluate(model), timestep # Initialize with given state seq if init_state_seq is not None: model.states_list[0].stateseq = init_state_seq for _ in xrange(100): model.resample_obs_distns() init_val = evaluate(model) vals, timesteps = zip(*[sample(model) for _ in progprint_xrange(N_iter)]) lls, plls, N_used, alphas, gammas, rates, obs_hypers = \ zip(*((init_val,) + vals)) timestamps = np.cumsum((0.,) + timesteps) return Results(name, lls, plls, N_used, alphas, gammas, rates, obs_hypers, model.copy_sample(), timestamps)
def fit_hmm(Xs, Xtest, N_samples=100): model = MultinomialHMM(K, D) for X in Xs: model.add_data(X) samples = [] lls = [] test_lls = [] pis = [] zs = [] timestamps = [time.time()] for smpl in progprint_xrange(N_samples): model.resample_model() timestamps.append(time.time()) samples.append(model.copy_sample()) # TODO: Use log_likelihood() to marginalize over z lls.append(model.log_likelihood()) # lls.append(model.log_likelihood_fixed_z()) test_lls.append(model.log_likelihood(Xtest)) # pis.append(testmodel.pis()[0]) zs.append(model.stateseqs[0]) lls = np.array(lls) test_lls = np.array(test_lls) pis = np.array(pis) zs = np.array(zs) timestamps = np.array(timestamps) timestamps -= timestamps[0] return model, lls, test_lls, pis, zs, timestamps
def fit_hmm(Xs, Xtest, D_hmm, N_samples=100): Nx = len(Xs) assert len(Xtest) == Nx print("Fitting HMM with %d states" % D_hmm) models = [MultinomialHMM(K, D_hmm, alpha_0=10.0) for _ in xrange(Nx)] for X, model in zip(Xs, models): model.add_data(X) def compute_pred_ll(): pred_ll = 0 for Xtr, Xte, model in zip(Xs, Xtest, models): pred_ll += model.log_likelihood(np.vstack((Xtr, Xte))) - model.log_likelihood(Xtr) return pred_ll init_results = (0, None, np.nan, np.nan, compute_pred_ll()) def resample(): tic = time.time() [model.resample_model() for model in models] toc = time.time() - tic return toc, None, np.nan, np.nan, compute_pred_ll() times, samples, lls, test_lls, pred_lls = \ map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples, perline=5)]))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
def train_model(model, train_data, test_data, N_samples=300, method='resample_model', thetas=None): print('Training %s with %s' % (model.__class__.__name__, method)) model.add_data(train_data) # Initialize to a given set of thetas if thetas is not None: model.thetas = thetas for d in model.documents: d.resample_z() init_like, init_perp, init_sample, init_time = \ model.log_likelihood(), model.perplexity(test_data), \ model.copy_sample(), time.time() def update(i): operator.methodcaller(method)(model) # print "ll: ", model.log_likelihood() return model.log_likelihood(), \ model.perplexity(test_data), \ model.copy_sample(), \ time.time() likes, perps, samples, timestamps = zip(*[update(i) for i in progprint_xrange(N_samples,perline=5)]) # Get relative timestamps timestamps = np.array((init_time,) + timestamps) timestamps -= timestamps[0] return Results((init_like,) + likes, (init_perp,) + perps, (init_sample,) + samples, timestamps)
def fit_hmm(Xs, Xtest, D_hmm, N_samples=100): print("Fitting HMM with %d states" % D_hmm) model = MultinomialHMM(K, D_hmm) for X in Xs: model.add_data(X) init_results = (0, None, model.log_likelihood(), model.log_likelihood(Xtest), (model.log_likelihood(np.vstack((Xs[0], Xtest))) - model.log_likelihood(Xs[0]))) def resample(): tic = time.time() model.resample_model() toc = time.time() - tic return toc, None, model.log_likelihood(), \ model.log_likelihood(Xtest), \ (model.log_likelihood(np.vstack((Xs[0], Xtest))) - model.log_likelihood(Xs[0])) times, samples, lls, test_lls, pred_lls = \ map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples)]))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
def fit(train_data, test_data, T, Niter, init_at_em, *args): resample = operator.methodcaller(method) def evaluate(model): ll, pll, perp = \ model.log_likelihood(), model.log_likelihood(test_data), \ model.perplexity(test_data) return ll, pll, perp def sample(model): tic = time.time() resample(model) timestep = time.time() - tic return evaluate(model), timestep print('Running %s...' % name) model = cls(train_data, T, *args) model = initializer(model) if init_at_em and initializer else model init_val = evaluate(model) vals, timesteps = zip(*[sample(model) for _ in progprint_xrange(Niter)]) lls, plls, perps = zip(*((init_val,) + vals)) timestamps = np.cumsum((0.,) + timesteps) return Results(lls, plls, perps, model.copy_sample(), timestamps)
def fit_lds_model(Xs, Xtest, D, N_samples=100): model = MultinomialLDS(K, D, init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1*np.eye(D)), dynamics_distn=AutoRegression(nu_0=D+1,S_0=1*np.eye(D),M_0=np.zeros((D,D)),K_0=1*np.eye(D)), sigma_C=0.01 ) for X in Xs: model.add_data(X) model.resample_parameters() init_results = (0, model, model.log_likelihood(), model.heldout_log_likelihood(Xtest, M=1), model.predictive_log_likelihood(Xtest, M=1000)) def resample(): tic = time.time() model.resample_model() toc = time.time() - tic return toc, None, model.log_likelihood(), \ model.heldout_log_likelihood(Xtest, M=1), \ model.predictive_log_likelihood(Xtest, M=1000) times, samples, lls, test_lls, pred_lls = \ map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples)]))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
def ais(self, N_samples=100, B=1000, steps_per_B=1, verbose=True, full_output=False, callback=None): """ Since Gibbs sampling as a function of temperature is implemented, we can use AIS to approximate the marginal likelihood of the model. """ # We use a linear schedule by default betas = np.linspace(0, 1, B) print "Estimating marginal likelihood with AIS" lw = np.zeros(N_samples) for m in progprint_xrange(N_samples): # Initialize the model with a draw from the prior self.initialize_from_prior() # Keep track of the log of the m-th weight # It starts at zero because the prior is assumed to be normalized lw[m] = 0.0 # Sample the intermediate distributions for b in xrange(1,B): if verbose: sys.stdout.write("M: %d\tBeta: %.3f \r" % (m,betas[b])) sys.stdout.flush() # Compute the ratio of this sample under this distribution # and the previous distribution. The difference is added # to the log weight curr_lp = self.log_probability(temperature=betas[b]) prev_lp = self.log_probability(temperature=betas[b-1]) lw[m] += curr_lp - prev_lp # Sample the model at temperature betas[b] # Take some number of steps per beta in hopes that # the Markov chain will reach equilibrium. for s in range(steps_per_B): self.collapsed_resample_model(temperature=betas[b]) # Call the given callback if callback: callback(self, m, b) if verbose: print "" print "W: %f" % lw[m] # Compute the mean of the weights to get an estimate of the normalization constant log_Z = -np.log(N_samples) + logsumexp(lw) # Use bootstrap to compute standard error subsamples = np.random.choice(lw, size=(100, N_samples), replace=True) log_Z_subsamples = logsumexp(subsamples, axis=1) - np.log(N_samples) std_log_Z = log_Z_subsamples.std() if full_output: return log_Z, std_log_Z, lw else: return log_Z, std_log_Z
def fit_gaussian_lds_model(Xs, Xtest, D_gauss_lds, N_samples=100): Nx = len(Xs) assert len(Xtest) == Nx print("Fitting Gaussian (Raw) LDS with %d states" % D_gauss_lds) from pylds.models import NonstationaryLDS models = [NonstationaryLDS( init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1*np.eye(D)), dynamics_distn=AutoRegression(nu_0=D+1,S_0=1*np.eye(D),M_0=np.zeros((D,D)),K_0=1*np.eye(D)), emission_distn=Regression(nu_0=K+1,S_0=K*np.eye(K),M_0=np.zeros((K,D)),K_0=K*np.eye(D))) for _ in xrange(Nx)] Xs_centered = [X - np.mean(X, axis=0)[None,:] + 1e-3*np.random.randn(*X.shape) for X in Xs] for X, model in zip(Xs_centered, models): model.add_data(X) def compute_pred_ll(): pred_ll = 0 for Xtr, Xte, model in zip(Xs_centered, Xtest, models): # Monte Carlo sample to get pi density implied by Gaussian LDS Npred = 10 Tpred = Xte.shape[0] preds = model.sample_predictions(Xtr, Tpred, Npred=Npred) # Convert predictions to a distribution by finding the # largest dimension for each predicted Gaussian. # Preds is T x K x Npred, inds is TxNpred inds = np.argmax(preds, axis=1) pi = np.array([np.bincount(inds[t], minlength=K) for t in xrange(Tpred)]) / float(Npred) assert np.allclose(pi.sum(axis=1), 1.0) pi = np.clip(pi, 1e-8, 1.0) pi /= pi.sum(axis=1)[:,None] # Compute the log likelihood under pi pred_ll += np.sum([Multinomial(weights=pi[t], K=K).log_likelihood(Xte[t][None,:]) for t in xrange(Tpred)]) return pred_ll # TODO: Get initial pred ll init_results = (0, None, np.nan, np.nan, compute_pred_ll()) def resample(): tic = time.time() [model.resample_model() for model in models] toc = time.time() - tic return toc, None, np.nan, np.nan, compute_pred_ll() times, samples, lls, test_lls, pred_lls = \ map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples, perline=5)]))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
def fit_gaussian_lds_model(Xs, Xtest, D_gauss_lds, N_samples=100): print("Fitting Gaussian (Raw) LDS with %d states" % D_gauss_lds) model = DefaultLDS(n=D_gauss_lds, p=K) Xs_centered = [X - np.mean(X, axis=0)[None,:] + 1e-3*np.random.randn(*X.shape) for X in Xs] for X in Xs_centered: model.add_data(X) # TODO: Get initial pred ll init_results = (0, None, np.nan, np.nan, np.nan) def resample(): tic = time.time() model.resample_model() toc = time.time() - tic # Monte Carlo sample to get pi density implied by Gaussian LDS Tpred = Xtest.shape[0] Npred = 1000 preds = model.sample_predictions(Xs_centered[0], Tpred, Npred=Npred) # Convert predictions to a distribution by finding the # largest dimension for each predicted Gaussian. # Preds is T x K x Npred, inds is TxNpred inds = np.argmax(preds, axis=1) pi = np.array([np.bincount(inds[t], minlength=K) for t in xrange(Tpred)]) / float(Npred) assert np.allclose(pi.sum(axis=1), 1.0) pi = np.clip(pi, 1e-8, 1.0) pi /= pi.sum(axis=1)[:,None] # Compute the log likelihood under pi pred_ll = np.sum([Multinomial(weights=pi[t], K=K).log_likelihood(Xtest[t][None,:]) for t in xrange(Tpred)]) return toc, None, np.nan, \ np.nan, \ pred_ll n_retries = 0 max_attempts = 5 while n_retries < max_attempts: try: times, samples, lls, test_lls, pred_lls = \ map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples)]))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps) except Exception as e: print("Caught exception: ", e.message) print("Retrying") n_retries += 1 raise Exception("Failed to fit the Raw Gaussian LDS model in %d attempts" % max_attempts)
def meanfield_coordinate_descent(self,tol=1e-1,maxiter=250,progprint=False,**kwargs): # NOTE: doesn't re-initialize! scores = [] step_iterator = xrange(maxiter) if not progprint else progprint_xrange(maxiter) for itr in step_iterator: scores.append(self.meanfield_coordinate_descent_step(**kwargs)) if scores[-1] is not None and len(scores) > 1: if np.abs(scores[-1]-scores[-2]) < tol: return scores print('WARNING: meanfield_coordinate_descent hit maxiter of %d' % maxiter) return scores
def fit_discrete_time_model_gibbs(S_dt, N_samples=100): # Now fit a DT model dt_model_test = pyhawkes.models.\ DiscreteTimeNetworkHawkesModelSpikeAndSlab(K=K, dt=dt, dt_max=dt_max, B=B, network_hypers=network_hypers) dt_model_test.add_data(S_dt) tic = time.time() for iter in progprint_xrange(N_samples, perline=25): dt_model_test.resample_model() toc = time.time() return (toc-tic) / N_samples
def fit_continuous_time_model_gibbs(S_ct, C_ct, N_samples=100): # Now fit a DT model ct_model = pyhawkes.models.\ ContinuousTimeNetworkHawkesModel(K, dt_max=dt_max, network_hypers=network_hypers) ct_model.add_data(S_ct, C_ct, T) tic = time.time() for iter in progprint_xrange(N_samples, perline=25): ct_model.resample_model() toc = time.time() return (toc-tic) / N_samples
def fit_lds_model(Xs, Xtest, D, N_samples=100): Nx = len(Xs) assert len(Xtest) == Nx mus = [X.sum(0) + 0.1 for X in Xs] mus = [mu / mu.sum() for mu in mus] # mus = [np.ones(K)/float(K) for _ in Xs] models = [ MultinomialLDS(K, D, init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1 * np.eye(D)), dynamics_distn=AutoRegression(nu_0=D + 1, S_0=1 * np.eye(D), M_0=np.zeros((D, D)), K_0=1 * np.eye(D)), sigma_C=1., mu_pi=mus[i]) for i in range(Nx) ] for X, model in zip(Xs, models): model.add_data(X) [model.resample_parameters() for model in models] def compute_pred_ll(): pred_ll = 0 for Xt, model in zip(Xtest, models): pred_ll += model.predictive_log_likelihood(Xt, M=1)[0] return pred_ll init_results = (0, models, np.nan, np.nan, compute_pred_ll()) def resample(): tic = time.time() [model.resample_model() for model in models] toc = time.time() - tic return toc, None, np.nan, np.nan, compute_pred_ll() times, samples, lls, test_lls, pred_lls = \ list(map(np.array, list(zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples, perline=5)]))))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
def fit_factor_analysis(y, mask=None, N_iters=100): print("Fitting Factor Analysis") model = FactorAnalysis(D_obs, D_latent) if mask is None: mask = np.ones_like(y, dtype=bool) # Center the data b = y.mean(0) data = model.add_data(y - b, mask=mask) for _ in progprint_xrange(N_iters): model.resample_model() C_init = np.column_stack((model.W, b)) return data.Z, C_init
def fit_ln_lds_model(Xs, Xtest, D, N_samples=100): """ Fit a logistic normal LDS model with pMCMC """ Nx = len(Xs) assert len(Xtest) == Nx print("Fitting Logistic Normal LDS with %d states" % D) mus = [X.sum(0) + 0.1 for X in Xs] mus = [np.log(mu/mu.sum()) for mu in mus] models = [LogisticNormalMultinomialLDS( init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1*np.eye(D)), dynamics_distn=AutoRegression(nu_0=D+1,S_0=D*np.eye(D),M_0=np.zeros((D,D)),K_0=D*np.eye(D)), emission_distn=Regression(nu_0=K+1,S_0=K*np.eye(K),M_0=np.zeros((K,D)),K_0=K*np.eye(D)), sigma_C=1.0, mu=mu) \ for mu in mus] for model in models: model.A = 0.5*np.eye(D) model.sigma_states = np.eye(D) model.C = 1.0*np.random.randn(K,D) model.sigma_obs = 0.1*np.eye(K) for X, model in zip(Xs, models): model.add_data(X) def compute_pred_ll(): pred_ll = 0 for Xte, model in zip(Xtest, models): pred_ll += model.predictive_log_likelihood(Xte, Npred=1)[0] return pred_ll init_results = (0, None, np.nan, np.nan, compute_pred_ll()) def resample(): tic = time.time() [model.resample_model() for model in models] toc = time.time() - tic return toc, None, np.nan, np.nan, compute_pred_ll() times, samples, lls, test_lls, pred_lls = \ map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples, perline=5)]))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
def fit_lds_model_with_pmcmc(Xs, Xtest, D, N_samples=100): """ Fit a logistic normal LDS model with pMCMC """ print("Fitting SBM-LDS with %d states using pMCMC" % D) model = ParticleSBMultinomialLDS( init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1 * np.eye(D)), dynamics_distn=AutoRegression(nu_0=D + 1, S_0=D * np.eye(D), M_0=np.zeros((D, D)), K_0=D * np.eye(D)), emission_distn=Regression(nu_0=K + 1, S_0=K * np.eye(K), M_0=np.zeros((K, D)), K_0=K * np.eye(D)), mu=pi_to_psi(np.ones(K) / K)) model.A = 0.5 * np.eye(D) model.sigma_states = np.eye(D) model.C = np.random.randn(K - 1, D) model.sigma_obs = 0.1 * np.eye(K) for X in Xs: model.add_data(X) compute_pred_ll = lambda: sum([ model.predictive_log_likelihood(Xt, data_index=i, Npred=10)[0] for i, Xt in enumerate(Xtest) ]) init_results = (0, None, model.log_likelihood(), np.nan, compute_pred_ll()) def resample(): tic = time.time() model.resample_model() toc = time.time() - tic # pred_ll = model.predictive_log_likelihood(Xtest, Npred=1000) return toc, None, model.log_likelihood(), \ np.nan, \ compute_pred_ll() times, samples, lls, test_lls, pred_lls = \ list(map(np.array, list(zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples)]))))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
def fit_lds_model(Xs, Xtest, D, N_samples=100): Nx = len(Xs) assert len(Xtest) == Nx model = MultinomialLDS(K, D, init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1 * np.eye(D)), dynamics_distn=AutoRegression(nu_0=D + 1, S_0=1 * np.eye(D), M_0=np.zeros((D, D)), K_0=1 * np.eye(D)), sigma_C=1.) for X in Xs: model.add_data(X) model.resample_parameters() compute_pred_ll = lambda: sum([ model.predictive_log_likelihood(Xt, data_index=i, M=10)[0] for i, Xt in enumerate(Xtest) ]) init_results = ( 0, None, model.log_likelihood(), # model.heldout_log_likelihood(Xtest, M=1), np.nan, compute_pred_ll()) def resample(): tic = time.time() model.resample_model() toc = time.time() - tic return toc, None, model.log_likelihood(), \ np.nan,\ compute_pred_ll() times, samples, lls, test_lls, pred_lls = \ list(map(np.array, list(zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples, perline=5)]))))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
def fit_ln_lds_model(Xs, Xtest, D, N_samples=100): """ Fit a logistic normal LDS model with pMCMC """ Nx = len(Xs) assert len(Xtest) == Nx print("Fitting Logistic Normal LDS with %d states" % D) mus = [X.sum(0) + 0.1 for X in Xs] mus = [np.log(mu / mu.sum()) for mu in mus] models = [LogisticNormalMultinomialLDS( init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1*np.eye(D)), dynamics_distn=AutoRegression(nu_0=D+1,S_0=D*np.eye(D),M_0=np.zeros((D,D)),K_0=D*np.eye(D)), emission_distn=Regression(nu_0=K+1,S_0=K*np.eye(K),M_0=np.zeros((K,D)),K_0=K*np.eye(D)), sigma_C=1.0, mu=mu) \ for mu in mus] for model in models: model.A = 0.5 * np.eye(D) model.sigma_states = np.eye(D) model.C = 1.0 * np.random.randn(K, D) model.sigma_obs = 0.1 * np.eye(K) for X, model in zip(Xs, models): model.add_data(X) def compute_pred_ll(): pred_ll = 0 for Xte, model in zip(Xtest, models): pred_ll += model.predictive_log_likelihood(Xte, Npred=1)[0] return pred_ll init_results = (0, None, np.nan, np.nan, compute_pred_ll()) def resample(): tic = time.time() [model.resample_model() for model in models] toc = time.time() - tic return toc, None, np.nan, np.nan, compute_pred_ll() times, samples, lls, test_lls, pred_lls = \ map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples, perline=5)]))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
def fit_gaussian_lds_model(Xs, N_samples=100): testmodel = DefaultLDS(n=D,p=K) for X in Xs: testmodel.add_data(X) samples = [] lls = [] for smpl in progprint_xrange(N_samples): testmodel.resample_model() samples.append(testmodel.copy_sample()) lls.append(testmodel.log_likelihood()) lls = np.array(lls) return lls
def fit_gaussian_lds_model(Xs, N_samples=100): testmodel = DefaultLDS(n=D, p=K) for X in Xs: testmodel.add_data(X) samples = [] lls = [] for smpl in progprint_xrange(N_samples): testmodel.resample_model() samples.append(testmodel.copy_sample()) lls.append(testmodel.log_likelihood()) lls = np.array(lls) return lls
def fit_lds_model(Xs, Xtest, N_samples=100): model = MultinomialLDS(K, D, init_dynamics_distn=Gaussian(mu_0=np.zeros(D), sigma_0=np.eye(D), kappa_0=1.0, nu_0=D + 1.0), dynamics_distn=AutoRegression(nu_0=D + 1, S_0=np.eye(D), M_0=np.zeros((D, D)), K_0=np.eye(D)), sigma_C=1) for X in Xs: model.add_data(X) data = model.data_list[0] samples = [] lls = [] test_lls = [] mc_test_lls = [] pis = [] psis = [] zs = [] timestamps = [time.time()] for smpl in progprint_xrange(N_samples): model.resample_model() timestamps.append(time.time()) samples.append(model.copy_sample()) # TODO: Use log_likelihood() to marginalize over z lls.append(model.log_likelihood()) # test_lls.append(model.heldout_log_likelihood(Xtest, M=50)[0]) mc_test_lls.append(model._mc_heldout_log_likelihood(Xtest, M=1)[0]) pis.append(model.pi(data)) psis.append(model.psi(data)) zs.append(data["states"].stateseq) lls = np.array(lls) test_lls = np.array(test_lls) pis = np.array(pis) psis = np.array(psis) zs = np.array(zs) timestamps = np.array(timestamps) timestamps -= timestamps[0] return model, lls, test_lls, mc_test_lls, pis, psis, zs, timestamps
def _EM_fit(self,method,tol=1e-1,maxiter=100,progprint=True): # NOTE: doesn't re-initialize! likes = [] step_iterator = xrange(maxiter) if not progprint else progprint_xrange(maxiter) for itr in step_iterator: method() likes.append(self.log_likelihood()) if len(likes) > 1: if likes[-1]-likes[-2] < tol: return likes elif likes[-1] < likes[-2]: # probably oscillation, do one more method() likes.append(self.log_likelihood()) return likes print 'WARNING: EM_fit reached maxiter of %d' % maxiter return likes
def fit_lds_model_with_pmcmc(Xs, Xtest, D, N_samples=100): """ Fit a logistic normal LDS model with pMCMC """ Nx = len(Xs) assert len(Xtest) == Nx print("Fitting SBM-LDS with %d states using pMCMC" % D) models = [ParticleSBMultinomialLDS( init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1*np.eye(D)), dynamics_distn=AutoRegression(nu_0=D+1,S_0=D*np.eye(D),M_0=np.zeros((D,D)),K_0=D*np.eye(D)), emission_distn=Regression(nu_0=K+1,S_0=K*np.eye(K),M_0=np.zeros((K,D)),K_0=K*np.eye(D)), mu=pi_to_psi(np.ones(K)/K), sigma_C=1.0) for _ in range(Nx)] for model in models: model.A = 0.5*np.eye(D) model.sigma_states = np.eye(D) model.C = np.random.randn(K-1,D) model.sigma_obs = 0.1*np.eye(K) for X, model in zip(Xs, models): model.add_data(X) def compute_pred_ll(): pred_ll = 0 for Xte, model in zip(Xtest, models): pred_ll += model.predictive_log_likelihood(Xte, Npred=100)[0] return pred_ll init_results = (0, None, np.nan, np.nan, compute_pred_ll()) def resample(): tic = time.time() [model.resample_model() for model in models] toc = time.time() - tic return toc, None, np.nan, np.nan, compute_pred_ll() times, samples, lls, test_lls, pred_lls = \ list(map(np.array, list(zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples, perline=5)]))))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
def _EM_fit(self,method,tol=1e-1,maxiter=100,progprint=False): # NOTE: doesn't re-initialize! likes = [] step_iterator = xrange(maxiter) if not progprint else progprint_xrange(maxiter) for itr in step_iterator: method() likes.append(self.log_likelihood()) if len(likes) > 1: if likes[-1]-likes[-2] < tol: return likes elif likes[-1] < likes[-2]: # probably oscillation, do one more method() likes.append(self.log_likelihood()) return likes print('WARNING: EM_fit reached maxiter of %d' % maxiter) return likes
def meanfield_coordinate_descent(self, tol=1e-1, maxiter=250, progprint=False, **kwargs): # NOTE: doesn't re-initialize! scores = [] step_iterator = xrange(maxiter) if not progprint else progprint_xrange( maxiter) for itr in step_iterator: scores.append(self.meanfield_coordinate_descent_step(**kwargs)) if scores[-1] is not None and len(scores) > 1: if np.abs(scores[-1] - scores[-2]) < tol: return scores print('WARNING: meanfield_coordinate_descent hit maxiter of %d' % maxiter) return scores
def em_example(true_model, X, Z_true, mask): # Fit a test model model = FactorAnalysis( D_obs, D_latent, # W=true_model.W, sigmasq=true_model.sigmasq ) inf_data = model.add_data(X, mask=mask) model.set_empirical_mean() lps = [] angles = [] N_iters = 100 for _ in progprint_xrange(N_iters): model.EM_step() lps.append(model.log_likelihood()) angles.append(principal_angle(true_model.W, model.W)) plot_results(lps, angles, Z_true, inf_data.E_Z)
def fit_ln_lds_model(Xs, Xtest, D, N_samples=100): """ Fit a logistic normal LDS model with pMCMC """ print("Fitting Logistic Normal LDS with %d states" % D) model = LogisticNormalMultinomialLDS( init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1 * np.eye(D)), dynamics_distn=AutoRegression(nu_0=D + 1, S_0=D * np.eye(D), M_0=np.zeros((D, D)), K_0=D * np.eye(D)), emission_distn=Regression(nu_0=K + 1, S_0=K * np.eye(K), M_0=np.zeros((K, D)), K_0=K * np.eye(D)), sigma_C=0.1) model.A = 0.5 * np.eye(D) model.sigma_states = np.eye(D) model.C = 0.33 * np.random.randn(K, D) model.sigma_obs = 0.1 * np.eye(K) for X in Xs: model.add_data(X) init_results = (0, None, model.log_likelihood(), np.nan, model.predictive_log_likelihood(Xtest, Npred=1000)) def resample(): tic = time.time() model.resample_model() toc = time.time() - tic pred_ll = model.predictive_log_likelihood(Xtest, Npred=1000) return toc, None, model.log_likelihood(), \ np.nan, \ pred_ll times, samples, lls, test_lls, pred_lls = \ map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples)]))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
def gibbs_example(true_model, X, Z_true, mask): # Fit a test model model = FactorAnalysis( D_obs, D_latent, # W=true_model.W, sigmasq=true_model.sigmasq ) inf_data = model.add_data(X, mask=mask) model.set_empirical_mean() lps = [] angles = [] N_iters = 100 for _ in progprint_xrange(N_iters): model.resample_model() lps.append(model.log_likelihood()) angles.append(principal_angle(true_model.W, model.W)) plot_results(lps, angles, Z_true, inf_data.Z)
def em_example(true_model, true_data): X, mask = true_data.X, true_data.mask # Fit a test model model = FactorAnalysis( D_obs, D_latent, # W=true_model.W, sigmasq=true_model.sigmasq ) inf_data = model.add_data(X, mask=mask) lps = [] angles = [] N_iters = 100 for _ in progprint_xrange(N_iters): model.EM_step() lps.append(model.log_likelihood()) angles.append(principal_angle(true_model.W, model.W)) plot_results(lps, angles, true_data.Z, inf_data.E_Z)
def meanfield_example(true_model, X, Z_true, mask): # Fit a test model model = FactorAnalysis( D_obs, D_latent, # W=true_model.W, sigmasq=true_model.sigmasq ) inf_data = model.add_data(X, mask=mask) model.set_empirical_mean() lps = [] angles = [] N_iters = 100 for _ in progprint_xrange(N_iters): model.meanfield_coordinate_descent_step() lps.append(model.expected_log_likelihood()) E_W, _, _, _ = model.regression.mf_expectations angles.append(principal_angle(true_model.W, E_W)) plot_results(lps, angles, Z_true, inf_data.Z)
def fit_lds_model(Xs, Xtest, D, N_samples=100): Nx = len(Xs) assert len(Xtest) == Nx mus = [X.sum(0) + 0.1 for X in Xs] mus = [mu/mu.sum() for mu in mus] # mus = [np.ones(K)/float(K) for _ in Xs] models = [MultinomialLDS(K, D, init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1*np.eye(D)), dynamics_distn=AutoRegression(nu_0=D+1,S_0=1*np.eye(D),M_0=np.zeros((D,D)),K_0=1*np.eye(D)), sigma_C=1., mu_pi=mus[i]) for i in xrange(Nx)] for X, model in zip(Xs, models): model.add_data(X) [model.resample_parameters() for model in models] def compute_pred_ll(): pred_ll = 0 for Xt, model in zip(Xtest, models): pred_ll += model.predictive_log_likelihood(Xt, M=1)[0] return pred_ll init_results = (0, models, np.nan, np.nan, compute_pred_ll()) def resample(): tic = time.time() [model.resample_model() for model in models] toc = time.time() - tic return toc, None, np.nan, np.nan, compute_pred_ll() times, samples, lls, test_lls, pred_lls = \ map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples, perline=5)]))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
def gibbs_example(true_model, true_data): X, mask = true_data.X, true_data.mask # Fit a test model model = FactorAnalysis( D_obs, D_latent, # W=true_model.W, sigmasq=true_model.sigmasq ) inf_data = model.add_data(X, mask=mask) lps = [] angles = [] N_iters = 100 for _ in progprint_xrange(N_iters): model.resample_model() lps.append(model.log_likelihood()) angles.append(principal_angle(true_model.W, model.W)) plot_results(lps, angles, true_data.Z, inf_data.Z)
def meanfield_example(true_model, true_data): X, mask = true_data.X, true_data.mask # Fit a test model model = FactorAnalysis( D_obs, D_latent, # W=true_model.W, sigmasq=true_model.sigmasq ) inf_data = model.add_data(X, mask=mask) lps = [] angles = [] N_iters = 100 for _ in progprint_xrange(N_iters): model.meanfield_coordinate_descent_step() lps.append(model.expected_log_likelihood()) E_W, _, _, _ = model.regression.mf_expectations angles.append(principal_angle(true_model.W, E_W)) plot_results(lps, angles, true_data.Z, inf_data.Z)
def fit_lds_model_with_pmcmc(Xs, Xtest, D, N_samples=100): """ Fit a logistic normal LDS model with pMCMC """ print("Fitting SBM-LDS with %d states using pMCMC" % D) model = ParticleSBMultinomialLDS( init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1*np.eye(D)), dynamics_distn=AutoRegression(nu_0=D+1,S_0=D*np.eye(D),M_0=np.zeros((D,D)),K_0=D*np.eye(D)), emission_distn=Regression(nu_0=K+1,S_0=K*np.eye(K),M_0=np.zeros((K,D)),K_0=K*np.eye(D)), mu=pi_to_psi(np.ones(K)/K)) model.A = 0.5*np.eye(D) model.sigma_states = np.eye(D) model.C = np.random.randn(K-1,D) model.sigma_obs = 0.1*np.eye(K) for X in Xs: model.add_data(X) compute_pred_ll = lambda: sum([model.predictive_log_likelihood(Xt, data_index=i, Npred=10)[0] for i,Xt in enumerate(Xtest)]) init_results = (0, None, model.log_likelihood(), np.nan, compute_pred_ll()) def resample(): tic = time.time() model.resample_model() toc = time.time() - tic # pred_ll = model.predictive_log_likelihood(Xtest, Npred=1000) return toc, None, model.log_likelihood(), \ np.nan, \ compute_pred_ll() times, samples, lls, test_lls, pred_lls = \ map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples)]))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
def fit_sbdtm_gibbs(train_data, test_data, timestamps, K, Niter, alpha_theta): def evaluate(model): ll, pll = model.log_likelihood(), model.log_likelihood(test_data) # print '{} '.format(ll), return ll, pll def sample(model): tic = time.time() model.resample() timestep = time.time() - tic return evaluate(model), timestep print "Running sbdtm gibbs..." model = StickbreakingDynamicTopicsLDA(train_data, timestamps, K, alpha_theta) init_val = evaluate(model) vals, timesteps = zip(*[sample(model) for _ in progprint_xrange(Niter)]) lls, plls = zip(*((init_val,) + vals)) times = np.cumsum((0,) + timesteps) return Results(lls, plls, model.copy_sample(), times)
def fit_lds_model(Xs, Xtest, N_samples=100): model = MultinomialLDS(K, D, init_dynamics_distn=Gaussian(mu_0=np.zeros(D), sigma_0=np.eye(D), kappa_0=1.0, nu_0=D+1.0), dynamics_distn=AutoRegression(nu_0=D+1,S_0=np.eye(D),M_0=np.zeros((D,D)),K_0=np.eye(D)), sigma_C=1 ) for X in Xs: model.add_data(X) data = model.data_list[0] samples = [] lls = [] test_lls = [] mc_test_lls = [] pis = [] psis = [] zs = [] timestamps = [time.time()] for smpl in progprint_xrange(N_samples): model.resample_model() timestamps.append(time.time()) samples.append(model.copy_sample()) # TODO: Use log_likelihood() to marginalize over z lls.append(model.log_likelihood()) # test_lls.append(model.heldout_log_likelihood(Xtest, M=50)[0]) mc_test_lls.append(model._mc_heldout_log_likelihood(Xtest, M=1)[0]) pis.append(model.pi(data)) psis.append(model.psi(data)) zs.append(data["states"].stateseq) lls = np.array(lls) test_lls = np.array(test_lls) pis = np.array(pis) psis = np.array(psis) zs = np.array(zs) timestamps = np.array(timestamps) timestamps -= timestamps[0] return model, lls, test_lls, mc_test_lls, pis, psis, zs, timestamps
def fit_arhmm(x, affine=True): print("Fitting Sticky ARHMM") dynamics_hypparams = \ dict(nu_0=D_latent + 2, S_0=np.eye(D_latent), M_0=np.hstack((np.eye(D_latent), np.zeros((D_latent, int(affine))))), K_0=np.eye(D_latent + affine), affine=affine) dynamics_hypparams = get_empirical_ar_params([x], dynamics_hypparams) dynamics_distns = [ AutoRegression(A=np.column_stack( (0.99 * np.eye(D_latent), np.zeros((D_latent, int(affine))))), sigma=np.eye(D_latent), **dynamics_hypparams) for _ in range(K) ] init_distn = Gaussian(nu_0=D_latent + 2, sigma_0=np.eye(D_latent), mu_0=np.zeros(D_latent), kappa_0=1.0) arhmm = ARWeakLimitStickyHDPHMM(init_state_distn='uniform', init_emission_distn=init_distn, obs_distns=dynamics_distns, alpha=3.0, kappa=10.0, gamma=3.0) arhmm.add_data(x) lps = [] for _ in progprint_xrange(1000): arhmm.resample_model() lps.append(arhmm.log_likelihood()) z_init = arhmm.states_list[0].stateseq z_init = np.concatenate(([0], z_init)) return arhmm, z_init
def fit_vb(name, model, test_data, N_iter=1000, init_state_seq=None): def evaluate(model): ll = model.log_likelihood() pll = model.log_likelihood(test_data) N_used = len(model.used_states) trans = model.trans_distn alpha = trans.alpha gamma = trans.gamma if hasattr(trans, "gamma") else None rates = model.rates.copy() obs_hypers = model.obs_hypers # print 'N_states: {}, \tPLL:{}\n'.format(len(model.used_states), pll), return ll, pll, N_used, alpha, gamma, rates, obs_hypers def sample(model): tic = time.time() model.meanfield_coordinate_descent_step() timestep = time.time() - tic # Resample from mean field posterior model._resample_from_mf() return evaluate(model), timestep # Initialize with given state seq if init_state_seq is not None: model.states_list[0].stateseq = init_state_seq for _ in xrange(100): model.resample_obs_distns() init_val = evaluate(model) vals, timesteps = zip(*[sample(model) for _ in progprint_xrange(200)]) lls, plls, N_used, alphas, gammas, rates, obs_hypers = \ zip(*((init_val,) + vals)) timestamps = np.cumsum((0.,) + timesteps) return Results(name, lls, plls, N_used, alphas, gammas, rates, obs_hypers, model.copy_sample(), timestamps)
def fit_ln_lds_model(Xs, Xtest, D, N_samples=100): """ Fit a logistic normal LDS model with pMCMC """ print("Fitting Logistic Normal LDS with %d states" % D) model = LogisticNormalMultinomialLDS( init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1*np.eye(D)), dynamics_distn=AutoRegression(nu_0=D+1,S_0=D*np.eye(D),M_0=np.zeros((D,D)),K_0=D*np.eye(D)), emission_distn=Regression(nu_0=K+1,S_0=K*np.eye(K),M_0=np.zeros((K,D)),K_0=K*np.eye(D)), sigma_C=0.1) model.A = 0.5*np.eye(D) model.sigma_states = np.eye(D) model.C = 0.33 * np.random.randn(K,D) model.sigma_obs = 0.1*np.eye(K) for X in Xs: model.add_data(X) init_results = (0, None, model.log_likelihood(), np.nan, model.predictive_log_likelihood(Xtest, Npred=1000)) def resample(): tic = time.time() model.resample_model() toc = time.time() - tic pred_ll = model.predictive_log_likelihood(Xtest, Npred=1000) return toc, None, model.log_likelihood(), \ np.nan, \ pred_ll times, samples, lls, test_lls, pred_lls = \ map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples)]))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
def fit_lds_model(Xs, Xtest, D, N_samples=100): Nx = len(Xs) assert len(Xtest) == Nx model = MultinomialLDS(K, D, init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1*np.eye(D)), dynamics_distn=AutoRegression(nu_0=D+1,S_0=1*np.eye(D),M_0=np.zeros((D,D)),K_0=1*np.eye(D)), sigma_C=1. ) for X in Xs: model.add_data(X) model.resample_parameters() compute_pred_ll = lambda: sum([model.predictive_log_likelihood(Xt, data_index=i, M=10)[0] for i,Xt in enumerate(Xtest)]) init_results = (0, None, model.log_likelihood(), # model.heldout_log_likelihood(Xtest, M=1), np.nan, compute_pred_ll()) def resample(): tic = time.time() model.resample_model() toc = time.time() - tic return toc, None, model.log_likelihood(), \ np.nan,\ compute_pred_ll() times, samples, lls, test_lls, pred_lls = \ map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples, perline=5)]))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
def max_likelihood(self, data, weights=None, stats=None): """ Maximize the likelihood for given data :param data: :param weights: :param stats: :return: """ if isinstance(data, list): x = np.vstack([d[0] for d in data]) y = np.vstack([d[1] for d in data]) elif isinstance(data, tuple): assert len(data) == 2 elif isinstance(data, np.ndarray): x, y = data[:, :self.D_in], data[:, self.D_in:] else: raise Exception("Invalid data type") from sklearn.linear_model import LogisticRegression for n in progprint_xrange(self.D_out): lr = LogisticRegression(fit_intercept=False) lr.fit(x, y[:, n]) self.A[n] = lr.coef_
def fit_sbdtm_gibbs(train_data, test_data, timestamps, K, Niter, alpha_theta): def evaluate(model): ll, pll = \ model.log_likelihood(), \ model.log_likelihood(test_data) # print '{} '.format(ll), return ll, pll def sample(model): tic = time.time() model.resample() timestep = time.time() - tic return evaluate(model), timestep print 'Running sbdtm gibbs...' model = StickbreakingDynamicTopicsLDA(train_data, timestamps, K, alpha_theta) init_val = evaluate(model) vals, timesteps = zip(*[sample(model) for _ in progprint_xrange(Niter)]) lls, plls = zip(*((init_val, ) + vals)) times = np.cumsum((0, ) + timesteps) return Results(lls, plls, model.copy_sample(), times)
del priormodel plt.figure() plt.plot(data[:, 0], data[:, 1], 'kx') plt.title('data') min_num_components, max_num_components = (1, 12) num_tries_each = 5 ### search over models using BIC as a model selection criterion BICs = [] examplemodels = [] for idx, num_components in enumerate( progprint_xrange(min_num_components, max_num_components + 1)): theseBICs = [] for i in xrange(num_tries_each): fitmodel = models.Mixture( alpha_0= 10000, # used for random initialization Gibbs sampling, big means use all components components=[ distributions.Gaussian(**obs_hypparams) for itr in range(num_components) ]) fitmodel.add_data(data) # use Gibbs sampling for initialization for itr in xrange(100): fitmodel.resample_model()
bkgd_hypers=true_model.bkgd_hypers, impulse_hypers=true_model.impulse_hypers, weight_hypers=true_model.weight_hypers) test_model.add_data(S) # Initialize with the standard model parameters if init_model is not None: test_model.initialize_with_standard_model(init_model) ########################################################### # Fit the test model with Gibbs sampling ########################################################### N_samples = 500 samples = [] lps = [] for itr in progprint_xrange(N_samples): lps.append(test_model.log_probability()) samples.append(test_model.copy_sample()) test_model.resample_model() ########################################################### # Analyze the samples ########################################################### N_samples = len(samples) A_samples = np.array([s.weight_model.A for s in samples]) W_samples = np.array([s.weight_model.W for s in samples]) g_samples = np.array([s.impulse_model.g for s in samples]) lambda0_samples = np.array([s.bias_model.lambda0 for s in samples]) lps = np.array(lps) offset = N_samples // 2
def fit_ct_network_hawkes_gibbs(S, S_test, dt, dt_max, output_path, model_args={}, standard_model=None, N_samples=100, time_limit=8 * 60 * 60): K = S.shape[1] S_ct, C_ct, T = convert_discrete_to_continuous(S, dt) S_test_ct, C_test_ct, T_test = convert_discrete_to_continuous(S_test, dt) # Check for existing Gibbs results if os.path.exists(output_path): with gzip.open(output_path, 'r') as f: print("Loading Gibbs results from ", output_path) results = pickle.load(f) else: print( "Fitting the data with a continuous time network Hawkes model using Gibbs sampling" ) test_model = \ ContinuousTimeNetworkHawkesModel(K, dt_max=dt_max, **model_args) test_model.add_data(S_ct, C_ct, T) # Initialize with the standard model parameters if standard_model is not None: test_model.initialize_with_standard_model(standard_model) # Gibbs sample samples = [] lps = [test_model.log_probability()] hlls = [ test_model.heldout_log_likelihood(S_test_ct, C_test_ct, T_test) ] times = [0] for _ in progprint_xrange(N_samples, perline=25): # Update the model tic = time.time() test_model.resample_model() times.append(time.time() - tic) samples.append(copy.deepcopy(test_model.get_parameters())) # Compute log probability and heldout log likelihood # lps.append(test_model.log_probability()) hlls.append( test_model.heldout_log_likelihood(S_test_ct, C_test_ct, T_test)) # # Save this sample # with open(output_path + ".gibbs.itr%04d.pkl" % itr, 'w') as f: # cPickle.dump(samples[-1], f, protocol=-1) # Check if time limit has been exceeded if np.sum(times) > time_limit: break # Get cumulative timestamps timestamps = np.cumsum(times) lps = np.array(lps) hlls = np.array(hlls) # Make results object results = Results(samples, timestamps, lps, hlls) # Save the Gibbs samples with gzip.open(output_path, 'w') as f: print("Saving Gibbs samples to ", output_path) pickle.dump(results, f, protocol=-1) return results
ct_model = pyhawkes.models.ContinuousTimeNetworkHawkesModel( K, dt_max=1., network_hypers=network_hypers) ct_model.add_data(S_ct, C_ct, T) # ct.resample_model() # Hard code parameters ct_model.bias_model.lambda0 = dt_model.bias_model.lambda0 ct_model.weight_model.A = dt_model.weight_model.A ct_model.weight_model.W = dt_model.weight_model.W print("CT LL: ", ct_model.heldout_log_likelihood(S_ct, C_ct, T)) # Fit the CT model ct_lls = [ct_model.log_likelihood()] N_samples = 100 for itr in progprint_xrange(N_samples, perline=25): ct_model.resample_model() ct_lls.append(ct_model.log_likelihood()) assert np.all(ct_model.weight_model.A == 1) # Now fit a DT model dt_model_test = pyhawkes.models.\ DiscreteTimeNetworkHawkesModelSpikeAndSlab(K=K, dt=dt, dt_max=dt_max, B=B, network_hypers=network_hypers) dt_model_test.add_data(S_dt) dt_lls = [] for itr in progprint_xrange(N_samples, perline=25): dt_model_test.resample_model() dt_lls.append(dt_model_test.log_likelihood()) assert np.all(dt_model_test.weight_model.A == 1)
emission_distn=BernoulliRegression(D_out=D_obs, D_in=D_latent + D_input)) model.add_data(data, inputs=inputs, stateseq=np.zeros((T, D_latent))) # Run a Gibbs sampler with Polya-gamma augmentation N_samples = 50 def gibbs_update(model): model.resample_model() smoothed_obs = model.states_list[0].smooth() ll = model.log_likelihood() return ll, model.states_list[0].gaussian_states, smoothed_obs lls_gibbs, x_smpls_gibbs, y_smooth_gibbs = \ zip(*[gibbs_update(model) for _ in progprint_xrange(N_samples)]) # Fit with a Bernoulli LDS using Laplace approximation for comparison model = DefaultBernoulliLDS(D_obs, D_latent, D_input=D_input, C=0.01 * np.random.randn(D_obs, D_latent), D=0.01 * np.random.randn(D_obs, D_input)) model.add_data(data, inputs=inputs, stateseq=np.zeros((T, D_latent))) N_iters = 50 def em_update(model): model.EM_step(verbose=True) smoothed_obs = model.states_list[0].smooth()
emission_distn=DiagonalRegression(D_obs, D_latent + D_input)) diag_model.add_data(data, inputs=inputs) # Also fit a model with a full covariance matrix full_model = DefaultLDS(D_obs, D_latent, D_input) full_model.add_data(data, inputs=inputs) # Fit with Gibbs sampling def update(model): model.resample_model() return model.log_likelihood() N_steps = 100 diag_lls = [update(diag_model) for _ in progprint_xrange(N_steps)] full_lls = [update(full_model) for _ in progprint_xrange(N_steps)] plt.figure() plt.plot([0, N_steps], truemodel.log_likelihood() * np.ones(2), '--k', label="true") plt.plot(diag_lls, label="diag cov.") plt.plot(full_lls, label="full cov.") plt.xlabel('iteration') plt.ylabel('log likelihood') plt.legend() # Predict forward in time T_given = 1800
def fit_network_hawkes_svi(S, S_test, dt, dt_max, output_path, model_args={}, standard_model=None, N_samples=100, time_limit=8 * 60 * 60, delay=10.0, forgetting_rate=0.25): T, K = S.shape # Check for existing Gibbs results if os.path.exists(output_path): with gzip.open(output_path, 'r') as f: print("Loading SVI results from ", output_path) results = pickle.load(f) else: print("Fitting the data with a network Hawkes model using SVI") test_model = DiscreteTimeNetworkHawkesModelGammaMixtureSBM( K=K, dt=dt, dt_max=dt_max, **model_args) test_model.add_data(S) # Initialize with the standard model parameters if standard_model is not None: test_model.initialize_with_standard_model(standard_model) # Precompute F_test F_test = test_model.basis.convolve_with_basis(S_test) # Initialize with the standard model parameters if standard_model is not None: test_model.initialize_with_standard_model(standard_model) # TODO: Add the data in minibatches minibatchsize = 3000 stepsize = (np.arange(N_samples) + delay)**(-forgetting_rate) # Stochastic variational inference samples = [] lps = [test_model.log_probability()] hlls = [test_model.heldout_log_likelihood(S_test)] times = [0] for itr in progprint_xrange(N_samples): # Update the model tic = time.time() test_model.sgd_step(minibatchsize=minibatchsize, stepsize=stepsize[itr]) times.append(time.time() - tic) # Resample from variational posterior to compute log prob and hlls test_model.resample_from_mf() # samples.append(test_model.copy_sample()) samples.append(copy.deepcopy(test_model.get_parameters())) # Compute log probability and heldout log likelihood # lps.append(test_model.log_probability()) hlls.append(test_model.heldout_log_likelihood(S_test, F=F_test)) # Save this sample # with open(output_path + ".svi.itr%04d.pkl" % itr, 'w') as f: # cPickle.dump(samples[-1], f, protocol=-1) # Check if time limit has been exceeded if np.sum(times) > time_limit: break # Get cumulative timestamps timestamps = np.cumsum(times) lps = np.array(lps) hlls = np.array(hlls) # Make results object results = Results(samples, timestamps, lps, hlls) # Save the Gibbs samples with gzip.open(output_path, 'w') as f: print("Saving SVI samples to ", output_path) pickle.dump(results, f, protocol=-1) return results
T = 50 dt = 1.0 dt_max = 3.0 network_hypers = {'c': np.array([0], dtype=np.int), 'p': 0.5, 'kappa': 3.0, 'v': 15.0} weight_hypers = {"kappa_0": 3.0, "nu_0": 15.0} model = DiscreteTimeNetworkHawkesModelGammaMixture(K=1, dt=dt, dt_max=dt_max, weight_hypers=weight_hypers, network_hypers=network_hypers) model.generate(T=T) # Gibbs sample and then generate new data N_samples = 10000 samples = [] lps = [] for itr in progprint_xrange(N_samples, perline=50): # Resample the model model.resample_model(resample_network=False) samples.append(model.copy_sample()) lps.append(model.log_probability()) # Geweke step model.data_list.pop() model.generate(T=T) # Compute sample statistics for second half of samples A_samples = np.array([s.weight_model.A for s in samples]) W_samples = np.array([s.weight_model.W for s in samples]) g_samples = np.array([s.impulse_model.g for s in samples]) lambda0_samples = np.array([s.bias_model.lambda0 for s in samples])
def _collect(r): ll = r.log_likelihood((X, y))[~inds].sum() err = ((y - r.predict(X))**2).sum(1) mse = np.mean(err[~inds]) return r.A.copy(), ll, mse def _update(r): r.resample([(X, y)]) return _collect(r) # Fit the standard regression smpls = [_collect(std_reg)] for _ in progprint_xrange(100): smpls.append(_update(std_reg)) smpls = zip(*smpls) std_As, std_lls, std_mses = tuple(map(np.array, smpls)) # Fit the robust regression smpls = [_collect(robust_reg)] for _ in progprint_xrange(100): smpls.append(_update(robust_reg)) smpls = zip(*smpls) robust_As, robust_lls, robust_mses = tuple(map(np.array, smpls)) # Plot the inferred regression function plt.figure(figsize=(8, 4)) xlim = (-3, 3) ylim = abs(y).max()
D = 10 # Number of documents V = 20 # Number of words N = 20 # Number of words per document alpha_beta = 1.0 # Generate synthetic data data = np.random.poisson(2, (D,V)) data = csr_matrix(data) # Sample a GP model = StickbreakingCorrelatedLDA(data, T, alpha_beta=alpha_beta) # Run a Geweke test thetas = [] betas = [] for itr in progprint_xrange(N_iter): # Resample the data model.generate(N, keep=True) # Resample the parameters model.resample() # Update our samples thetas.append(model.theta.copy()) betas.append(model.beta.copy()) # Check that the PG-Multinomial samples are distributed like the prior thetas = np.array(thetas) theta_mean = thetas.mean(0) theta_std = thetas.std(0)
kappa_0=1.0, nu_0=D_in+1), dynamics_distn=AutoRegression( sigma=sigma_states, nu_0=D_in+1, S_0=D_in*np.eye(D_in), M_0=np.zeros((D_in, D_in)), K_0=D_in*np.eye(D_in)), emission_distn=PGEmissions(D_out, D_in, C=C, sigmasq_C=1.0)) model.add_data(data.X) N_samples = 1000 def update(model): model.resample_model() z_inf = model.states_list[0].stateseq C_inf = model.C psi_inf = z_inf.dot(C_inf.T) p_inf = logistic(psi_inf) return model.log_likelihood(), p_inf results = [update(model) for _ in progprint_xrange(N_samples)] lls = np.array([r[0] for r in results]) ps = np.array([r[1] for r in results]) plt.figure() plt.plot(data.X, 'bx', ls="none") # plt.plot(psi, 'r') plt.plot(p_true, 'r', lw=2) plt.errorbar(np.arange(T), ps.mean(0), yerr=ps.std(0), fmt='--r', ) plt.ylim(-0.1, data.X.max() + .1) plt.figure() plt.plot(lls) plt.xlabel("Iteration") plt.ylabel("LL") plt.show()
def fit_gaussian_lds_model(Xs, Xtest, D_gauss_lds, N_samples=100): Nx = len(Xs) assert len(Xtest) == Nx print("Fitting Gaussian (Raw) LDS with %d states" % D_gauss_lds) from pylds.models import NonstationaryLDS models = [ NonstationaryLDS( init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1 * np.eye(D)), dynamics_distn=AutoRegression(nu_0=D + 1, S_0=1 * np.eye(D), M_0=np.zeros((D, D)), K_0=1 * np.eye(D)), emission_distn=Regression(nu_0=K + 1, S_0=K * np.eye(K), M_0=np.zeros((K, D)), K_0=K * np.eye(D))) for _ in xrange(Nx) ] Xs_centered = [ X - np.mean(X, axis=0)[None, :] + 1e-3 * np.random.randn(*X.shape) for X in Xs ] for X, model in zip(Xs_centered, models): model.add_data(X) def compute_pred_ll(): pred_ll = 0 for Xtr, Xte, model in zip(Xs_centered, Xtest, models): # Monte Carlo sample to get pi density implied by Gaussian LDS Npred = 10 Tpred = Xte.shape[0] preds = model.sample_predictions(Xtr, Tpred, Npred=Npred) # Convert predictions to a distribution by finding the # largest dimension for each predicted Gaussian. # Preds is T x K x Npred, inds is TxNpred inds = np.argmax(preds, axis=1) pi = np.array( [np.bincount(inds[t], minlength=K) for t in xrange(Tpred)]) / float(Npred) assert np.allclose(pi.sum(axis=1), 1.0) pi = np.clip(pi, 1e-8, 1.0) pi /= pi.sum(axis=1)[:, None] # Compute the log likelihood under pi pred_ll += np.sum([ Multinomial(weights=pi[t], K=K).log_likelihood(Xte[t][None, :]) for t in xrange(Tpred) ]) return pred_ll # TODO: Get initial pred ll init_results = (0, None, np.nan, np.nan, compute_pred_ll()) def resample(): tic = time.time() [model.resample_model() for model in models] toc = time.time() - tic return toc, None, np.nan, np.nan, compute_pred_ll() times, samples, lls, test_lls, pred_lls = \ map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples, perline=5)]))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps)
D_obs = 1 D_latent = 2 D_input = 0 T = 2000 # Simulate from one LDS truemodel = DefaultLDS(D_obs, D_latent, D_input) inputs = np.random.randn(T, D_input) data, stateseq = truemodel.generate(T, inputs=inputs) # Fit with another LDS model = DefaultLDS(D_obs, D_latent, D_input) model.add_data(data, inputs=inputs) # Initialize with a few iterations of Gibbs for _ in progprint_xrange(10): model.resample_model() # Run EM def update(model): model.EM_step() return model.log_likelihood() lls = [update(model) for _ in progprint_xrange(50)] # Plot the log likelihoods plt.figure() plt.plot(lls) plt.xlabel('iteration')
def fit_gaussian_lds_model(Xs, Xtest, D_gauss_lds, N_samples=100): print("Fitting Gaussian (Raw) LDS with %d states" % D_gauss_lds) model = DefaultLDS(n=D_gauss_lds, p=K) Xs_centered = [ X - np.mean(X, axis=0)[None, :] + 1e-3 * np.random.randn(*X.shape) for X in Xs ] for X in Xs_centered: model.add_data(X) # TODO: Get initial pred ll init_results = (0, None, np.nan, np.nan, np.nan) def resample(): tic = time.time() model.resample_model() toc = time.time() - tic # Monte Carlo sample to get pi density implied by Gaussian LDS Tpred = Xtest.shape[0] Npred = 1000 preds = model.sample_predictions(Xs_centered[0], Tpred, Npred=Npred) # Convert predictions to a distribution by finding the # largest dimension for each predicted Gaussian. # Preds is T x K x Npred, inds is TxNpred inds = np.argmax(preds, axis=1) pi = np.array( [np.bincount(inds[t], minlength=K) for t in xrange(Tpred)]) / float(Npred) assert np.allclose(pi.sum(axis=1), 1.0) pi = np.clip(pi, 1e-8, 1.0) pi /= pi.sum(axis=1)[:, None] # Compute the log likelihood under pi pred_ll = np.sum([ Multinomial(weights=pi[t], K=K).log_likelihood(Xtest[t][None, :]) for t in xrange(Tpred) ]) return toc, None, np.nan, \ np.nan, \ pred_ll n_retries = 0 max_attempts = 5 while n_retries < max_attempts: try: times, samples, lls, test_lls, pred_lls = \ map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples)]))) timestamps = np.cumsum(times) return Results(lls, test_lls, pred_lls, samples, timestamps) except Exception as e: print("Caught exception: ", e.message) print("Retrying") n_retries += 1 raise Exception("Failed to fit the Raw Gaussian LDS model in %d attempts" % max_attempts)