예제 #1
0
    def __init__(self, K, n, C=None, sigma_C=1, mu=None, mu_pi=None):
        """
        Create a PGMultinomial distribution with mean and covariance for psi.

        :param K:       Dimensionality of the multinomial distribution
        :param mu_C:    Mean of the matrix normal distribution over C
        """
        assert isinstance(K, int) and K >= 2, "K must be an integer >= 2"
        self.K = K

        assert isinstance(n, int) and n >= 1, "n must be an integer >= 1"
        self.n = n

        # Initialize emission matrix C
        self.sigma_C = sigma_C
        if C is None:
            self.C = self.sigma_C * np.random.randn(self.K-1, self.n)
            # mu, sigma = compute_psi_cmoments(np.ones(K))
            # self.C = compute_psi_cmoments(np.ones(K))[0][:,None] * np.ones((self.K-1, self.n))
        else:
            assert C.shape == (self.K-1, self.n)
            self.C = C

        # Initialize the observation mean (mu)
        if mu is None and mu_pi is None:
            self.mu = np.zeros(self.K-1)
        elif mu is not None:
            assert mu.shape == (self.K-1,)
            self.mu = mu
        else:
            assert mu_pi.shape == (self.K,)
            self.mu = pi_to_psi(mu_pi)

        # Initialize Polya-gamma augmentation variables
        self.ppgs = initialize_polya_gamma_samplers()
예제 #2
0
파일: gp.py 프로젝트: yinsenm/pgmult
    def add_data(self, Z, X, optimize_hypers=True):

        assert Z.ndim == 2 and Z.shape[1] == self.D
        M = Z.shape[0]
        assert X.shape == (M, self.K)

        # Get the empirical probabilities (offset by 1 to ensure nonzero)
        pi_emp_train = (self.alpha+X).astype(np.float) / \
                       (self.alpha + X).sum(axis=1)[:,None]

        # Convert these to psi's
        self.Z = Z
        self.psi = np.array([pi_to_psi(pi) for pi in pi_emp_train])

        # Compute the mean value of psi
        self.mu = self.psi.mean(axis=0)
        self.psi -= self.mu

        # Create the GP Regression model
        from GPy.models import GPRegression
        self.model = GPRegression(Z, self.psi, self.kernel)

        # Optimize the kernel parameters
        if optimize_hypers:
            self.model.optimize(messages=True)
예제 #3
0
파일: census_gp.py 프로젝트: HIPS/pgmult
def fit_gp_multinomial_model(model, test, pi_train=None, N_samples=100, run=1):
    if pi_train is not None:
        if isinstance(model, pgmult.gp.LogisticNormalGP):
            model.data_list[0]["psi"] = ln_pi_to_psi(pi_train) - model.mu
        elif isinstance(model, pgmult.gp.MultinomialGP):
            model.data_list[0]["psi"] = pi_to_psi(pi_train) - model.mu
            model.resample_omega()
    else:
        model.initialize_from_data()

    ### Inference
    results_base = os.path.join("results", "names", "run%03d" % run, "results")
    results_file = results_base + ".pkl.gz"
    if os.path.exists(results_file):
        with gzip.open(results_file, "r") as f:
            samples, lls, pred_lls, timestamps = pickle.load(f)

    else:
        Z_test = get_inputs(test)
        lls = [model.log_likelihood()]
        samples = [model.copy_sample()]
        pred_ll, pred_pi = model.predictive_log_likelihood(Z_test, test.data)
        pred_lls = [pred_ll]
        pred_pis = [pred_pi]
        times = [0]

        # Print initial values
        print("Initial LL: ", lls[0])
        print("Initial Pred LL: ", pred_lls[0])


        for itr in range(N_samples):
            print("Iteration ", itr)
            tic = time.time()
            model.resample_model(verbose=True)
            times.append(time.time()-tic)

            samples.append(model.copy_sample())
            lls.append(model.log_likelihood())
            pred_ll, pred_pi = model.predictive_log_likelihood(get_inputs(test), test.data)
            pred_lls.append(pred_ll)
            pred_pis.append(pred_pi)

            print("Log likelihood: ", lls[-1])
            print("Pred Log likelihood: ", pred_ll)

            # Save this sample
            # with gzip.open(results_file + ".itr%03d.pkl.gz" % itr, "w") as f:
            #     pickle.dump(model, f, protocol=-1)

        lls = np.array(lls)
        pred_lls = np.array(pred_lls)
        timestamps = np.cumsum(times)

    return samples, lls, pred_lls, pred_pis, timestamps
예제 #4
0
def fit_gp_multinomial_model(model, test, pi_train=None, N_samples=100, run=1):
    if pi_train is not None:
        if isinstance(model, pgmult.gp.LogisticNormalGP):
            model.data_list[0]["psi"] = ln_pi_to_psi(pi_train) - model.mu
        elif isinstance(model, pgmult.gp.MultinomialGP):
            model.data_list[0]["psi"] = pi_to_psi(pi_train) - model.mu
            model.resample_omega()
    else:
        model.initialize_from_data()

    ### Inference
    results_base = os.path.join("results", "names", "run%03d" % run, "results")
    results_file = results_base + ".pkl.gz"
    if os.path.exists(results_file):
        with gzip.open(results_file, "r") as f:
            samples, lls, pred_lls, timestamps = pickle.load(f)

    else:
        Z_test = get_inputs(test)
        lls = [model.log_likelihood()]
        samples = [model.copy_sample()]
        pred_ll, pred_pi = model.predictive_log_likelihood(Z_test, test.data)
        pred_lls = [pred_ll]
        pred_pis = [pred_pi]
        times = [0]

        # Print initial values
        print("Initial LL: ", lls[0])
        print("Initial Pred LL: ", pred_lls[0])


        for itr in xrange(N_samples):
            print("Iteration ", itr)
            tic = time.time()
            model.resample_model(verbose=True)
            times.append(time.time()-tic)

            samples.append(model.copy_sample())
            lls.append(model.log_likelihood())
            pred_ll, pred_pi = model.predictive_log_likelihood(get_inputs(test), test.data)
            pred_lls.append(pred_ll)
            pred_pis.append(pred_pi)

            print("Log likelihood: ", lls[-1])
            print("Pred Log likelihood: ", pred_ll)

            # Save this sample
            # with gzip.open(results_file + ".itr%03d.pkl.gz" % itr, "w") as f:
            #     pickle.dump(model, f, protocol=-1)

        lls = np.array(lls)
        pred_lls = np.array(pred_lls)
        timestamps = np.cumsum(times)

    return samples, lls, pred_lls, pred_pis, timestamps
예제 #5
0
def fit_lds_model_with_pmcmc(Xs, Xtest, D, N_samples=100):
    """
    Fit a logistic normal LDS model with pMCMC
    """
    Nx = len(Xs)
    assert len(Xtest) == Nx

    print("Fitting SBM-LDS with %d states using pMCMC" % D)
    models = [
        ParticleSBMultinomialLDS(
            init_dynamics_distn=GaussianFixed(mu=np.zeros(D),
                                              sigma=1 * np.eye(D)),
            dynamics_distn=AutoRegression(nu_0=D + 1,
                                          S_0=D * np.eye(D),
                                          M_0=np.zeros((D, D)),
                                          K_0=D * np.eye(D)),
            emission_distn=Regression(nu_0=K + 1,
                                      S_0=K * np.eye(K),
                                      M_0=np.zeros((K, D)),
                                      K_0=K * np.eye(D)),
            mu=pi_to_psi(np.ones(K) / K),
            sigma_C=1.0) for _ in xrange(Nx)
    ]

    for model in models:
        model.A = 0.5 * np.eye(D)
        model.sigma_states = np.eye(D)
        model.C = np.random.randn(K - 1, D)
        model.sigma_obs = 0.1 * np.eye(K)

    for X, model in zip(Xs, models):
        model.add_data(X)

    def compute_pred_ll():
        pred_ll = 0
        for Xte, model in zip(Xtest, models):
            pred_ll += model.predictive_log_likelihood(Xte, Npred=100)[0]
        return pred_ll

    init_results = (0, None, np.nan, np.nan, compute_pred_ll())

    def resample():
        tic = time.time()
        [model.resample_model() for model in models]
        toc = time.time() - tic

        return toc, None, np.nan, np.nan, compute_pred_ll()

    times, samples, lls, test_lls, pred_lls = \
        map(np.array, zip(*([init_results] +
            [resample() for _ in progprint_xrange(N_samples, perline=5)])))
    timestamps = np.cumsum(times)

    return Results(lls, test_lls, pred_lls, samples, timestamps)
예제 #6
0
def test_psi_pi_conversion_3d():
    K = 10
    N = 10
    D = 10

    pi = np.random.rand(N, D, K)
    pi /= np.sum(pi, axis=2, keepdims=True)
    psi = np.array([pi_to_psi(p) for p in pi])
    pi2 = psi_to_pi(psi, axis=2)

    assert np.allclose(pi, pi2), "Mapping is not invertible."
예제 #7
0
def test_psi_pi_conversion():
    K = 10

    pi = np.ones(K) / float(K)
    psi = pi_to_psi(pi)
    pi2 = psi_to_pi(psi)

    print("pi:  ", pi)
    print("psi: ", psi)
    print("pi2: ", pi2)

    assert np.allclose(pi, pi2), "Mapping is not invertible."
예제 #8
0
def test_psi_pi_conversion():
    K = 10

    pi = np.ones(K) / float(K)
    psi = pi_to_psi(pi)
    pi2 = psi_to_pi(psi)

    print("pi:  ", pi)
    print("psi: ", psi)
    print("pi2: ", pi2)

    assert np.allclose(pi, pi2), "Mapping is not invertible."
예제 #9
0
파일: gp.py 프로젝트: yinsenm/pgmult
    def initialize_from_data(self, pi_lim=None, initialize_to_mle=True):
        """
        Initialize the psi's to the empirical mean of the data
        :return:
        """
        for data in self.data_list:
            # Compute the empirical probability
            X = data["X"]
            M = data["M"]
            assert X.ndim == 2 and X.shape[1] == self.K

            # Get the empirical probabilities (offset by 1 to ensure nonzero)
            alpha = 1
            pi_emp = (alpha+X).astype(np.float) / \
                           (alpha + X).sum(axis=1)[:,None]
            pi_emp_mean = pi_emp.mean(axis=0)


            # Set mu equal to the empirical mean value of pi
            psi_emp_mean = pi_to_psi(pi_emp_mean)
            self.mu = psi_emp_mean
            # self.mu = np.zeros(self.K-1)

            if initialize_to_mle:
                # Convert empirical values to psi
                psi_emp = np.array([pi_to_psi(p) for p in pi_emp])
                psi_emp -= self.mu
                assert psi_emp.shape == (M, self.K-1)

                # Set the deviations from the mean to zero
                data["psi"] = psi_emp
            else:
                data["psi"] = np.zeros((M, self.K-1))

        # Resample the omegas
        self.resample_omega()
예제 #10
0
파일: ap_lds.py 프로젝트: yinsenm/pgmult
def fit_lds_model_with_pmcmc(Xs, Xtest, D, N_samples=100):
    """
    Fit a logistic normal LDS model with pMCMC
    """
    print("Fitting SBM-LDS with %d states using pMCMC" % D)
    model = ParticleSBMultinomialLDS(
        init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1 * np.eye(D)),
        dynamics_distn=AutoRegression(nu_0=D + 1,
                                      S_0=D * np.eye(D),
                                      M_0=np.zeros((D, D)),
                                      K_0=D * np.eye(D)),
        emission_distn=Regression(nu_0=K + 1,
                                  S_0=K * np.eye(K),
                                  M_0=np.zeros((K, D)),
                                  K_0=K * np.eye(D)),
        mu=pi_to_psi(np.ones(K) / K))

    model.A = 0.5 * np.eye(D)
    model.sigma_states = np.eye(D)
    model.C = np.random.randn(K - 1, D)
    model.sigma_obs = 0.1 * np.eye(K)

    for X in Xs:
        model.add_data(X)

    compute_pred_ll = lambda: sum([
        model.predictive_log_likelihood(Xt, data_index=i, Npred=10)[0]
        for i, Xt in enumerate(Xtest)
    ])

    init_results = (0, None, model.log_likelihood(), np.nan, compute_pred_ll())

    def resample():
        tic = time.time()
        model.resample_model()
        toc = time.time() - tic

        # pred_ll = model.predictive_log_likelihood(Xtest, Npred=1000)

        return toc, None, model.log_likelihood(), \
            np.nan, \
            compute_pred_ll()

    times, samples, lls, test_lls, pred_lls = \
        list(map(np.array, list(zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples)])))))
    timestamps = np.cumsum(times)

    return Results(lls, test_lls, pred_lls, samples, timestamps)
예제 #11
0
def fit_lds_model_with_pmcmc(Xs, Xtest, D, N_samples=100):
    """
    Fit a logistic normal LDS model with pMCMC
    """
    Nx = len(Xs)
    assert len(Xtest) == Nx

    print("Fitting SBM-LDS with %d states using pMCMC" % D)
    models = [ParticleSBMultinomialLDS(
                init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1*np.eye(D)),
                dynamics_distn=AutoRegression(nu_0=D+1,S_0=D*np.eye(D),M_0=np.zeros((D,D)),K_0=D*np.eye(D)),
                emission_distn=Regression(nu_0=K+1,S_0=K*np.eye(K),M_0=np.zeros((K,D)),K_0=K*np.eye(D)),
                mu=pi_to_psi(np.ones(K)/K),
                sigma_C=1.0)
             for _ in xrange(Nx)]

    for model in models:
        model.A = 0.5*np.eye(D)
        model.sigma_states = np.eye(D)
        model.C = np.random.randn(K-1,D)
        model.sigma_obs = 0.1*np.eye(K)

    for X, model in zip(Xs, models):
        model.add_data(X)

    def compute_pred_ll():
        pred_ll = 0
        for Xte, model in zip(Xtest, models):
            pred_ll += model.predictive_log_likelihood(Xte, Npred=100)[0]
        return pred_ll

    init_results = (0, None, np.nan, np.nan, compute_pred_ll())

    def resample():
        tic = time.time()
        [model.resample_model() for model in models]
        toc = time.time() - tic

        return toc, None, np.nan, np.nan, compute_pred_ll()

    times, samples, lls, test_lls, pred_lls = \
        map(np.array, zip(*([init_results] +
            [resample() for _ in progprint_xrange(N_samples, perline=5)])))
    timestamps = np.cumsum(times)

    return Results(lls, test_lls, pred_lls, samples, timestamps)
예제 #12
0
파일: ap_lds.py 프로젝트: fivejjs/pgmult
def fit_lds_model_with_pmcmc(Xs, Xtest, D, N_samples=100):
    """
    Fit a logistic normal LDS model with pMCMC
    """
    print("Fitting SBM-LDS with %d states using pMCMC" % D)
    model = ParticleSBMultinomialLDS(
        init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1*np.eye(D)),
        dynamics_distn=AutoRegression(nu_0=D+1,S_0=D*np.eye(D),M_0=np.zeros((D,D)),K_0=D*np.eye(D)),
        emission_distn=Regression(nu_0=K+1,S_0=K*np.eye(K),M_0=np.zeros((K,D)),K_0=K*np.eye(D)),
        mu=pi_to_psi(np.ones(K)/K))

    model.A = 0.5*np.eye(D)
    model.sigma_states = np.eye(D)
    model.C = np.random.randn(K-1,D)
    model.sigma_obs = 0.1*np.eye(K)

    for X in Xs:
        model.add_data(X)

    compute_pred_ll = lambda: sum([model.predictive_log_likelihood(Xt, data_index=i, Npred=10)[0]
                                   for i,Xt in enumerate(Xtest)])

    init_results = (0, None, model.log_likelihood(),
                    np.nan, compute_pred_ll())

    def resample():
        tic = time.time()
        model.resample_model()
        toc = time.time() - tic

        # pred_ll = model.predictive_log_likelihood(Xtest, Npred=1000)

        return toc, None, model.log_likelihood(), \
            np.nan, \
            compute_pred_ll()

    times, samples, lls, test_lls, pred_lls = \
        map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples)])))
    timestamps = np.cumsum(times)

    return Results(lls, test_lls, pred_lls, samples, timestamps)
예제 #13
0
파일: ctm.py 프로젝트: fivejjs/pgmult
    nonempty_docs = np.asarray(model.data.sum(1) > 0).ravel()
    model.theta[nonempty_docs] = ln_psi_to_pi(lmbda)
    model.resample_z()
    return model


fit_lda_gibbs = sampler_fitter(
    'fit_lda_gibbs', StandardLDA, 'resample', lda_initializer)
fit_lda_collapsed = sampler_fitter(
    'fit_lda_collapsed', StandardLDA, 'resample_collapsed', lda_initializer)
fit_lnctm_gibbs = sampler_fitter(
    'fit_lnctm_gibbs', LogisticNormalCorrelatedLDA, 'resample',
    make_ctm_initializer(lambda lmbda: lmbda))
fit_sbctm_gibbs = sampler_fitter(
    'fit_sbctm_gibbs', StickbreakingCorrelatedLDA, 'resample',
    make_ctm_initializer(lambda lmbda: pi_to_psi(ln_psi_to_pi(lmbda))))


########################
#  inspecting results  #
########################

def plot_sb_interpretable_results(sb_results, words):
    nwords = 5
    Sigma = sb_results[-1][-1]
    T = Sigma.shape[0]

    def get_topwords(topic):
        return words[np.argsort(sb_results[-1][0][:,topic])[-nwords:]]

    lim = np.abs(Sigma).max()
예제 #14
0
 def pi(self, value):
     self.psi = pi_to_psi(value)
예제 #15
0
    nonempty_docs = np.asarray(model.data.sum(1) > 0).ravel()
    model.theta[nonempty_docs] = ln_psi_to_pi(lmbda)
    model.resample_z()
    return model


fit_lda_gibbs = sampler_fitter(
    'fit_lda_gibbs', StandardLDA, 'resample', lda_initializer)
fit_lda_collapsed = sampler_fitter(
    'fit_lda_collapsed', StandardLDA, 'resample_collapsed', lda_initializer)
fit_lnctm_gibbs = sampler_fitter(
    'fit_lnctm_gibbs', LogisticNormalCorrelatedLDA, 'resample',
    make_ctm_initializer(lambda lmbda: lmbda))
fit_sbctm_gibbs = sampler_fitter(
    'fit_sbctm_gibbs', StickbreakingCorrelatedLDA, 'resample',
    make_ctm_initializer(lambda lmbda: pi_to_psi(ln_psi_to_pi(lmbda))))


########################
#  inspecting results  #
########################

def plot_sb_interpretable_results(sb_results, words):
    nwords = 5
    Sigma = sb_results[-1][-1]
    T = Sigma.shape[0]

    def get_topwords(topic):
        return words[np.argsort(sb_results[-1][0][:,topic])[-nwords:]]

    lim = np.abs(Sigma).max()
예제 #16
0
파일: lda.py 프로젝트: fivejjs/pgmult
 def theta(self, theta):
     self.psi = pi_to_psi(theta)
예제 #17
0
 def theta(self, theta):
     self.psi = pi_to_psi(theta)