def lnL_pca_gauss(delta_pk, Pk):
''' Gaussian pseudo-likelihood calculated using PCA decomposition.
i.e. the data vector is decomposed in PCA components so that
L = p(x_pca,0)p(x_pca,1)...p(x_pca,n). Each p(x_pca,i) is estimated using
N(0, sig_pca,i).
Notes
-----
- By construction this evaluates the same likelihood as the
Gaussian functional pseudo-likelihood. It was implemented by convenience.
'''
X, mu_X = meansub(Pk) # mean subtract
X_pca, W_pca = whiten(X, method='pca') # whitened data
var_pca = np.zeros(X_pca.shape[1])
for i in range(X_pca.shape[1]):
var_pca[i] = np.var(X_pca[:, i])
cov = np.diag(var_pca)
ggg = multinorm(np.zeros(len(mu_X)), cov)
if len(delta_pk.shape) == 1:
x_obv = np.dot(delta_pk, W_pca)
else:
x_obv = np.zeros(delta_pk.shape)
for i_obv in range(delta_pk.shape[0]):
x_obv[i_obv, :] = np.dot(delta_pk[i_obv, :], W_pca)
return np.log(ggg.pdf(x_obv))
def test_8gaussians3d_box(self):
box = list(product(range(2), repeat=3))
data = []
for i, count in enumerate(8 * [5000]):
means3, sigma = (box[i], 0.25 + ((i + 1) / 8 + sum(box[i])) / 10)
covar3 = sigma**2 * identity(3)
for v in multinorm(means3, covar3).rvs(count):
data.append((i, v))
mi_est = wke(data).calculate_mi(k=NN_K)
mi_accur = 1.72251 # calculated in Mathematica for continuous distributions
self.assertAlmostEqual((mi_est - mi_accur) / mi_accur, 0, delta=RTOL)
def test_8gaussians3d_snake(self):
data = []
for dist_i, count in enumerate(8 * [5000]):
mu, sigma = dist_i + 1, 33 + 3 * dist_i
means3 = [mu**2.5, 50 * cos(mu / 0.75), 200 * sin(mu / 1.5)]
covar3 = sigma**2 * identity(3)
for v in multinorm(means3, covar3).rvs(count):
data.append((dist_i, v))
mi_est = wke(data).calculate_mi(k=NN_K)
mi_accur = 1.85959 # calculated in Mathematica for continuous distributions
self.assertAlmostEqual((mi_est - mi_accur) / mi_accur, 0, delta=RTOL)
def sample_mcmc(mcmc, recalculate, sd_stop_after, scaling_stop_after, save_path):
# print("start", mcmc['name'])
compute_scaling_factor = len(mcmc['chain']) < scaling_stop_after
compute_new_sd = len(mcmc['chain']) < sd_stop_after
dists = mcmc['dists']
# print(mcmc['name'])
print()
for iteration in trange(recalculate, desc=mcmc['name'], leave=False, position=0):
# for iteration in tqdm(range(recalculate), desc=mcmc['name'], leave=False, position=0):
# for iteration in range(recalculate):
# print (iteration, mcmc['name'])
if not (mcmc['active']): # When G-R converges, stop running all but one. -conition on GRB
continue
logger.info(' ' * 20 + 'CHAIN {} ITERATION {}'.format(mcmc['name'], len(mcmc['accepted'])))
# Save Chain
if len(mcmc['chain']) % 20 == 0:
save_mcmc(mcmc, save_path)
# Change parameters in warm up phase
if iteration == recalculate - 1:
# acceptance rate
accept_star = np.mean(mcmc['accepted'][-recalculate:])
mcmc['rates'] = np.append(mcmc['rates'], accept_star)
# new scaling factor - pg. 24
if compute_scaling_factor:
new_scaling_factor = mcmc['scaling_factor'][-1]
# new_scaling_factor *= np.e ** ((accept_star - mcmc['accept_hat']) / len(mcmc['scaling_factor']))
new_scaling_factor *= np.e ** (accept_star - mcmc['accept_hat'])
mcmc['scaling_factor'] = np.append(mcmc['scaling_factor'], new_scaling_factor)
else:
new_scaling_factor = 1
# new cov pg. 24
if compute_new_sd:
new_cov_tmp = mcmc['cov'].copy()
try:
sigma_star = np.cov(mcmc['chain'][-recalculate:, :].T)
new_cov = mcmc['cov'].copy() * 0.25 + 0.75 * sigma_star
proposed = multinorm(mcmc['values'], new_cov * new_scaling_factor ** 2)
mcmc['cov'] = new_cov
except Exception as e:
print (e)
print("Singular COV at", len(mcmc['accepted']), mcmc['name'])
print(mcmc['cov'])
mcmc['cov'] = new_cov_tmp
mcmc['sd'] = new_scaling_factor ** 2 * mcmc['cov']
# Current Stats
ll_now = log_liklihood(mcmc['y_now_M'], mcmc['datay1'], mcmc['sigma'])
ll_now += log_liklihood(mcmc['y2_now_M'], mcmc['datay2'], mcmc['sigma2'])
# Pick new set
try:
proposed = multinorm(mcmc['values'], mcmc['sd']) # Try new set ***************************************
except Exception as e:
print(e)
print(mcmc['name'], "TUNRED OFF", len(mcmc['accepted']))
mcmc['active'] = False
save_mcmc(mcmcs[grj], save_path)
continue
guess = proposed.rvs()
g = mcmc['initial_guess']
g[mcmc['active_params']] = guess
# Check if possible set
g[1] %= 2 * np.pi # Phi is special case #CHANGE THIS IF OMEGA NOT INCLUDED/NOT
test = [dists[dist].pdf(guess[i]) <= 0 for i, dist in enumerate(mcmc['active_params'])]
logger.info(str(g))
if any(test): # If test fails, LL is -inf
logger.error('Bad prior {} {}'.format(guess, test))
ll_star = -np.inf
y_star_M, y2_star_M, state_z = -np.inf * np.ones((3, 192)),-np.inf * np.ones(36), -np.inf * np.ones(270)
else: # We can carry on with MCMC
try:
y_star_M, y2_star_M, state_z = run_model(mcmc['state_0'], mcmc['start'], mcmc['end'], *g, e=1,
r_0=40) # RUN MODEL =============================================
# logger.info(str(y_star_M))
ll_star = log_liklihood(y_star_M, mcmc['datay1'], mcmc['sigma'])
ll_star += log_liklihood(y2_star_M, mcmc['datay2'], mcmc['sigma2'])
logger.info(str(ll_star))
if ll_star < -99999: # Something bad happend BY DESIGN at model
logger.warning('bad set for model {} with guess {}'.format(mcmc['name'], iteration))
except:
logger.error('exception at model {} PROBABLY S-I-R fail'.format(mcmc['name']))
ll_star = -np.inf
# when possible, run model
log_r = ll_star - ll_now
draw = np.random.rand()
if log_r > np.log(draw):
mcmc['values'] = guess.copy()
mcmc['accepted'] = np.append(mcmc['accepted'], 1)
mcmc['y_now_M'] = y_star_M
mcmc['y2_now_M'] = y2_star_M
else:
mcmc['accepted'] = np.append(mcmc['accepted'], 0)
# Update
mcmc['chain'] = np.vstack((mcmc['chain'], mcmc['values']))
mcmc['guesses'] = np.vstack((mcmc['guesses'], guess))
mcmc['y_hat_M'] = np.concatenate((mcmc['y_hat_M'], y_star_M[None, :, :]), axis=0)
# print ('>>>>>>>>>>>>>>>')
# print (mcmc['y2_hat_M'])
# print (y2_star_M)
mcmc['y2_hat_M'] = np.vstack((mcmc['y2_hat_M'], y2_star_M))
mcmc['max_likelihood'] = np.max((mcmc['max_likelihood'], ll_now))
mcmc['ll'] = np.vstack((mcmc['ll'], np.array([ll_now, ll_star])))
mcmc['state_z'] = np.vstack((mcmc['state_z'], state_z))
return mcmc
def sample_single(self, recalculate=500):
compute_scaling_factor = self.scaling_stop_after > len(self)
compute_sd = self.sd_stop_after > len(self)
for iteration in trange(recalculate, desc=self.name, leave=False, position=0):
if not self.active: continue
logger.info(' ' * 20 + 'CHAIN {} ITERATION {}'.format(self.name, len(self.accepted)))
# Save Chain
self.autosave(50)
if iteration == recalculate - 1:
# Acceptance rate
accept_star = np.mean(self.accepted[-recalculate:])
self.rates = np.append(self.rates, accept_star)
# New scaling factor
if compute_scaling_factor:
new_scaling_factor = self.scaling_factor[-1]
new_scaling_factor *= np.e ** (accept_star - self.accept_hat)
self.scaling_factor = np.append(self.scaling_factor, new_scaling_factor)
else:
new_scaling_factor = 1
# New COV
if compute_sd:
new_cov_tmp = self.cov.copy()
try:
sigma_star = np.cov(self.chain[-recalculate:, :].T)
new_cov = self.cov.copy() * 0.25 + 0.75 * sigma_star
proposed = multinorm(self.values, new_cov * new_scaling_factor ** 2)
self.cov = new_cov
except Exception as e:
print(e)
print("Singular COV at", len(self), self.name)
print(self.cov)
self.cov = new_cov_tmp
self.sd = new_scaling_factor ** 2 * self.cov
# Current State
ll_now = self.ll_now()
# tmp = self.y_now
# ptmp = tmp / tmp.sum(axis=0)
# try:
# proposed = multinorm(self.values, self.sd)
# except Exception as e:
# print(e)
# print(self.name, "TURNED OFF", len(self))
# self.turn_off()
# save_mcmc(self)
# continue
proposed = multinorm(self.values, self.sd)
guess = proposed.rvs()
# print(guess)
self.model.update(guess)
if not self.model.check_proposal(): # Bad guess
logger.info("bad prior")
ll_star = -np.inf
y_star, state_z = self.no_likelihood
self.model.rollback()
else: # Good guess continue MCMC
try:
self.set_stochastics()
logger.info('guess: {}'.format(guess))
y_star, state_z = self.run_model()
# ll_star = log_likelihood(y_star, self.ydata, self.sigma)
ll_star = self.ll_model(y_star)
logger.info(str(ll_star))
if ll_star < -(1e20):
logger.warning('bad set for model {} with guess {}'.format(self.name, iteration))
except Exception as e:
logger.info(e)
logger.error('exception at model {} PROBABLY S-I-R fail'.format(self.name))
y_star, state_z = self.no_likelihood
ll_star = -np.inf
log_r = ll_star - ll_now
draw = np.random.rand()
if log_r > np.log(draw):
self.values = self.model.values
self.accepted = np.append(self.accepted, 1)
self.y_now = y_star
else:
self.accepted = np.append(self.accepted, 0)
# Update
self.chain = np.vstack((self.chain, self.values))
self.guesses = np.vstack((self.guesses, guess))
self.mle = np.max((self.mle, ll_now))
self.yhat_history = np.concatenate((self.yhat_history, y_star[None, :, :]), axis=0)
self.state_z_history.append(self.state_z)
self.ll_history = np.vstack((self.ll_history, np.array([ll_now, ll_star])))