def lnL_pca_gauss(delta_pk, Pk): ''' Gaussian pseudo-likelihood calculated using PCA decomposition. i.e. the data vector is decomposed in PCA components so that L = p(x_pca,0)p(x_pca,1)...p(x_pca,n). Each p(x_pca,i) is estimated using N(0, sig_pca,i). Notes ----- - By construction this evaluates the same likelihood as the Gaussian functional pseudo-likelihood. It was implemented by convenience. ''' X, mu_X = meansub(Pk) # mean subtract X_pca, W_pca = whiten(X, method='pca') # whitened data var_pca = np.zeros(X_pca.shape[1]) for i in range(X_pca.shape[1]): var_pca[i] = np.var(X_pca[:, i]) cov = np.diag(var_pca) ggg = multinorm(np.zeros(len(mu_X)), cov) if len(delta_pk.shape) == 1: x_obv = np.dot(delta_pk, W_pca) else: x_obv = np.zeros(delta_pk.shape) for i_obv in range(delta_pk.shape[0]): x_obv[i_obv, :] = np.dot(delta_pk[i_obv, :], W_pca) return np.log(ggg.pdf(x_obv))
def test_8gaussians3d_box(self): box = list(product(range(2), repeat=3)) data = [] for i, count in enumerate(8 * [5000]): means3, sigma = (box[i], 0.25 + ((i + 1) / 8 + sum(box[i])) / 10) covar3 = sigma**2 * identity(3) for v in multinorm(means3, covar3).rvs(count): data.append((i, v)) mi_est = wke(data).calculate_mi(k=NN_K) mi_accur = 1.72251 # calculated in Mathematica for continuous distributions self.assertAlmostEqual((mi_est - mi_accur) / mi_accur, 0, delta=RTOL)
def test_8gaussians3d_snake(self): data = [] for dist_i, count in enumerate(8 * [5000]): mu, sigma = dist_i + 1, 33 + 3 * dist_i means3 = [mu**2.5, 50 * cos(mu / 0.75), 200 * sin(mu / 1.5)] covar3 = sigma**2 * identity(3) for v in multinorm(means3, covar3).rvs(count): data.append((dist_i, v)) mi_est = wke(data).calculate_mi(k=NN_K) mi_accur = 1.85959 # calculated in Mathematica for continuous distributions self.assertAlmostEqual((mi_est - mi_accur) / mi_accur, 0, delta=RTOL)
def sample_mcmc(mcmc, recalculate, sd_stop_after, scaling_stop_after, save_path): # print("start", mcmc['name']) compute_scaling_factor = len(mcmc['chain']) < scaling_stop_after compute_new_sd = len(mcmc['chain']) < sd_stop_after dists = mcmc['dists'] # print(mcmc['name']) print() for iteration in trange(recalculate, desc=mcmc['name'], leave=False, position=0): # for iteration in tqdm(range(recalculate), desc=mcmc['name'], leave=False, position=0): # for iteration in range(recalculate): # print (iteration, mcmc['name']) if not (mcmc['active']): # When G-R converges, stop running all but one. -conition on GRB continue logger.info(' ' * 20 + 'CHAIN {} ITERATION {}'.format(mcmc['name'], len(mcmc['accepted']))) # Save Chain if len(mcmc['chain']) % 20 == 0: save_mcmc(mcmc, save_path) # Change parameters in warm up phase if iteration == recalculate - 1: # acceptance rate accept_star = np.mean(mcmc['accepted'][-recalculate:]) mcmc['rates'] = np.append(mcmc['rates'], accept_star) # new scaling factor - pg. 24 if compute_scaling_factor: new_scaling_factor = mcmc['scaling_factor'][-1] # new_scaling_factor *= np.e ** ((accept_star - mcmc['accept_hat']) / len(mcmc['scaling_factor'])) new_scaling_factor *= np.e ** (accept_star - mcmc['accept_hat']) mcmc['scaling_factor'] = np.append(mcmc['scaling_factor'], new_scaling_factor) else: new_scaling_factor = 1 # new cov pg. 24 if compute_new_sd: new_cov_tmp = mcmc['cov'].copy() try: sigma_star = np.cov(mcmc['chain'][-recalculate:, :].T) new_cov = mcmc['cov'].copy() * 0.25 + 0.75 * sigma_star proposed = multinorm(mcmc['values'], new_cov * new_scaling_factor ** 2) mcmc['cov'] = new_cov except Exception as e: print (e) print("Singular COV at", len(mcmc['accepted']), mcmc['name']) print(mcmc['cov']) mcmc['cov'] = new_cov_tmp mcmc['sd'] = new_scaling_factor ** 2 * mcmc['cov'] # Current Stats ll_now = log_liklihood(mcmc['y_now_M'], mcmc['datay1'], mcmc['sigma']) ll_now += log_liklihood(mcmc['y2_now_M'], mcmc['datay2'], mcmc['sigma2']) # Pick new set try: proposed = multinorm(mcmc['values'], mcmc['sd']) # Try new set *************************************** except Exception as e: print(e) print(mcmc['name'], "TUNRED OFF", len(mcmc['accepted'])) mcmc['active'] = False save_mcmc(mcmcs[grj], save_path) continue guess = proposed.rvs() g = mcmc['initial_guess'] g[mcmc['active_params']] = guess # Check if possible set g[1] %= 2 * np.pi # Phi is special case #CHANGE THIS IF OMEGA NOT INCLUDED/NOT test = [dists[dist].pdf(guess[i]) <= 0 for i, dist in enumerate(mcmc['active_params'])] logger.info(str(g)) if any(test): # If test fails, LL is -inf logger.error('Bad prior {} {}'.format(guess, test)) ll_star = -np.inf y_star_M, y2_star_M, state_z = -np.inf * np.ones((3, 192)),-np.inf * np.ones(36), -np.inf * np.ones(270) else: # We can carry on with MCMC try: y_star_M, y2_star_M, state_z = run_model(mcmc['state_0'], mcmc['start'], mcmc['end'], *g, e=1, r_0=40) # RUN MODEL ============================================= # logger.info(str(y_star_M)) ll_star = log_liklihood(y_star_M, mcmc['datay1'], mcmc['sigma']) ll_star += log_liklihood(y2_star_M, mcmc['datay2'], mcmc['sigma2']) logger.info(str(ll_star)) if ll_star < -99999: # Something bad happend BY DESIGN at model logger.warning('bad set for model {} with guess {}'.format(mcmc['name'], iteration)) except: logger.error('exception at model {} PROBABLY S-I-R fail'.format(mcmc['name'])) ll_star = -np.inf # when possible, run model log_r = ll_star - ll_now draw = np.random.rand() if log_r > np.log(draw): mcmc['values'] = guess.copy() mcmc['accepted'] = np.append(mcmc['accepted'], 1) mcmc['y_now_M'] = y_star_M mcmc['y2_now_M'] = y2_star_M else: mcmc['accepted'] = np.append(mcmc['accepted'], 0) # Update mcmc['chain'] = np.vstack((mcmc['chain'], mcmc['values'])) mcmc['guesses'] = np.vstack((mcmc['guesses'], guess)) mcmc['y_hat_M'] = np.concatenate((mcmc['y_hat_M'], y_star_M[None, :, :]), axis=0) # print ('>>>>>>>>>>>>>>>') # print (mcmc['y2_hat_M']) # print (y2_star_M) mcmc['y2_hat_M'] = np.vstack((mcmc['y2_hat_M'], y2_star_M)) mcmc['max_likelihood'] = np.max((mcmc['max_likelihood'], ll_now)) mcmc['ll'] = np.vstack((mcmc['ll'], np.array([ll_now, ll_star]))) mcmc['state_z'] = np.vstack((mcmc['state_z'], state_z)) return mcmc
def sample_single(self, recalculate=500): compute_scaling_factor = self.scaling_stop_after > len(self) compute_sd = self.sd_stop_after > len(self) for iteration in trange(recalculate, desc=self.name, leave=False, position=0): if not self.active: continue logger.info(' ' * 20 + 'CHAIN {} ITERATION {}'.format(self.name, len(self.accepted))) # Save Chain self.autosave(50) if iteration == recalculate - 1: # Acceptance rate accept_star = np.mean(self.accepted[-recalculate:]) self.rates = np.append(self.rates, accept_star) # New scaling factor if compute_scaling_factor: new_scaling_factor = self.scaling_factor[-1] new_scaling_factor *= np.e ** (accept_star - self.accept_hat) self.scaling_factor = np.append(self.scaling_factor, new_scaling_factor) else: new_scaling_factor = 1 # New COV if compute_sd: new_cov_tmp = self.cov.copy() try: sigma_star = np.cov(self.chain[-recalculate:, :].T) new_cov = self.cov.copy() * 0.25 + 0.75 * sigma_star proposed = multinorm(self.values, new_cov * new_scaling_factor ** 2) self.cov = new_cov except Exception as e: print(e) print("Singular COV at", len(self), self.name) print(self.cov) self.cov = new_cov_tmp self.sd = new_scaling_factor ** 2 * self.cov # Current State ll_now = self.ll_now() # tmp = self.y_now # ptmp = tmp / tmp.sum(axis=0) # try: # proposed = multinorm(self.values, self.sd) # except Exception as e: # print(e) # print(self.name, "TURNED OFF", len(self)) # self.turn_off() # save_mcmc(self) # continue proposed = multinorm(self.values, self.sd) guess = proposed.rvs() # print(guess) self.model.update(guess) if not self.model.check_proposal(): # Bad guess logger.info("bad prior") ll_star = -np.inf y_star, state_z = self.no_likelihood self.model.rollback() else: # Good guess continue MCMC try: self.set_stochastics() logger.info('guess: {}'.format(guess)) y_star, state_z = self.run_model() # ll_star = log_likelihood(y_star, self.ydata, self.sigma) ll_star = self.ll_model(y_star) logger.info(str(ll_star)) if ll_star < -(1e20): logger.warning('bad set for model {} with guess {}'.format(self.name, iteration)) except Exception as e: logger.info(e) logger.error('exception at model {} PROBABLY S-I-R fail'.format(self.name)) y_star, state_z = self.no_likelihood ll_star = -np.inf log_r = ll_star - ll_now draw = np.random.rand() if log_r > np.log(draw): self.values = self.model.values self.accepted = np.append(self.accepted, 1) self.y_now = y_star else: self.accepted = np.append(self.accepted, 0) # Update self.chain = np.vstack((self.chain, self.values)) self.guesses = np.vstack((self.guesses, guess)) self.mle = np.max((self.mle, ll_now)) self.yhat_history = np.concatenate((self.yhat_history, y_star[None, :, :]), axis=0) self.state_z_history.append(self.state_z) self.ll_history = np.vstack((self.ll_history, np.array([ll_now, ll_star])))