def elbo_grad(z_sample, mu, sigma_sq, y, X, P, prior_sigma): score_mu = (z_sample - mu)/(sigma_sq) score_logsigma_sq = (-1/(2*sigma_sq) + np.power((z_sample - mu),2)/(2*np.power(sigma_sq,2))) * sigma_sq log_p = np.sum(y * np.log(sigmoid(np.dot(X,z_sample))) + (1-y) * np.log(1-sigmoid(np.dot(X,z_sample))))\ + np.sum(norm.logpdf(z_sample, np.zeros(P), prior_sigma*np.ones(P))) log_q = np.sum(norm.logpdf(z_sample, mu, np.sqrt(sigma_sq))) return np.concatenate([score_mu,score_logsigma_sq])*(log_p - log_q)
def test_gauss(): # sample from N(0,1) pdf = lambda x: norm.logpdf(x) condpdf = lambda a, b: norm.logpdf(b, loc=a) condsamp = lambda a: np.random.normal(loc=a) def sampler(x0, niters): x = x0 for _ in xrange(niters): x = sample(x, pdf, condpdf, condsamp) yield x bins = np.linspace(-3, 3, 1000) smoothing = 1e-5 actual_samples = np.random.normal(size=1000) actual_hist = hist(actual_samples, bins) + smoothing actual_hist /= actual_hist.sum() slice_samples = np.array(list(sampler(1.0, 10000))[::10]) slice_hist = hist(slice_samples, bins) + smoothing slice_hist /= slice_hist.sum() assert KL_approx(actual_hist, slice_hist, bins[1] - bins[0]) <= 0.1
def ln_posterior(p, coordinate, obs_vlos, err_vlos): pm_ra,pm_dec,vlos = p vgal = gc.vhel_to_gal(coordinate, pm=(pm_ra*u.mas/u.yr,pm_dec*u.mas/u.yr), rv=vlos*u.km/u.s, vcirc=vcirc, vlsr=vlsr, galactocentric_frame=gc_frame) vtot = np.sqrt(np.sum(vgal**2)).to(u.km/u.s).value return norm.logpdf(vtot, loc=0., scale=sigma_halo) + norm.logpdf(vlos, loc=obs_vlos, scale=err_vlos)
def get_normal_example(sample_count): loc = 1.0 scale = 2.0 samples0 = norm.rvs(loc, scale, sample_count) samples1 = norm.rvs(loc, scale, sample_count) scores0 = norm.logpdf(samples0, loc, scale) scores1 = norm.logpdf(samples1, loc, scale) samples = numpy.array(zip(samples0, samples1)) scores = scores0 + scores1 return {'name': 'normal', 'samples': samples, 'scores': scores}
def test_aic_posterior_dependence(): d = norm.rvs(size=1000) p = norm.logpdf(d) p2 = norm.logpdf(d, scale=2) c = ChainConsumer() c.add_chain(d, posterior=p, num_free_params=1, num_eff_data_points=1000) c.add_chain(d, posterior=p2, num_free_params=1, num_eff_data_points=1000) aics = c.comparison.aic() assert len(aics) == 2 assert aics[0] == 0 expected = 2 * np.log(2) assert np.isclose(aics[1], expected, atol=1e-3)
def test_dic_posterior_dependence(): d = norm.rvs(size=1000000) p = norm.logpdf(d) p2 = norm.logpdf(d, scale=2) c = ChainConsumer() c.add_chain(d, posterior=p) c.add_chain(d, posterior=p2) bics = c.comparison.dic() assert len(bics) == 2 assert bics[1] == 0 dic1 = 2 * np.mean(-2 * p) + 2 * norm.logpdf(0) dic2 = 2 * np.mean(-2 * p2) + 2 * norm.logpdf(0, scale=2) assert np.isclose(bics[0], dic1 - dic2, atol=1e-3)
def log_prior(self): """ Compute the prior probability of F, mu0, and lmbda """ lp = 0 # Log prior of F under spherical Gaussian prior from scipy.stats import norm lp += norm.logpdf(self.L, 0, np.sqrt(self.sigma)).sum() # Log prior of mu_0 and mu_self lp += norm.logpdf(self.mu_0, 0, 1) lp += norm.logpdf(self.mu_self, 0, 1) return lp
def calculate_bias(chain_dictionary, supernovae, cosmologies, return_mbs=False): supernovae = supernovae[supernovae[:, 6] > 0.0] supernovae = supernovae[supernovae[:, 0] < 10.3] masses = np.ones(supernovae.shape[0]) redshifts = supernovae[:, 0] apparents = supernovae[:, 1] stretches = supernovae[:, 2] colours = supernovae[:, 3] smear = supernovae[:, 4] apparents += smear # return np.ones(chain_dictionary["weight"].shape) existing_prob = norm.logpdf(colours, 0, 0.1) + norm.logpdf(stretches, 0, 1) + norm.logpdf(smear, 0, 0.1) weight = [] for i in range(chain_dictionary["mean_MB"].size): om = np.round(chain_dictionary["Om"][i], decimals=3) key = "%0.3f" % om mus = cosmologies[key](redshifts) dscale = chain_dictionary["dscale"][i] dratio = chain_dictionary["dratio"][i] redshift_pre_comp = 0.9 + np.power(10, 0.95 * redshifts) mass_correction = dscale * (1.9 * (1 - dratio) / redshift_pre_comp + dratio) mass_correction = 0 mabs = apparents - mus + chain_dictionary["alpha"][i] * stretches - chain_dictionary["beta"][i] * colours + mass_correction * masses mbx1cs = np.vstack((mabs, stretches, colours)).T chain_MB = chain_dictionary["mean_MB"][i] chain_x1 = chain_dictionary["mean_x1"][i] chain_c = chain_dictionary["mean_c"][i] chain_sigmas = np.array([chain_dictionary["sigma_MB"][i], chain_dictionary["sigma_x1"][i], chain_dictionary["sigma_c"][i]]) chain_sigmas_mat = np.dot(chain_sigmas[:, None], chain_sigmas[None, :]) chain_correlations = np.dot(chain_dictionary["intrinsic_correlation"][i], chain_dictionary["intrinsic_correlation"][i].T) chain_pop_cov = chain_correlations * chain_sigmas_mat chain_mean = np.array([chain_MB, chain_x1, chain_c]) chain_prob = multivariate_normal.logpdf(mbx1cs, chain_mean, chain_pop_cov) reweight = logsumexp(chain_prob - existing_prob) if reweight < 1: for key in chain_dictionary.keys(): print(key, chain_dictionary[key][i]) weight.append(reweight) weights = np.array(weight) if return_mbs: mean_mb = chain_dictionary["mean_MB"] - chain_dictionary["alpha"] * chain_dictionary["mean_x1"] + \ chain_dictionary["beta"] * chain_dictionary["mean_c"] return weights, mean_mb return weights
def proba(self, name, data, given): if self.mean_ is None or self.covariance_ is None: self.compute_mean_cov_matrix() ll = 0 Q = self.precision_ idx = self.indices[name] evidence_indices = np.array([self.indices[n] for n in given if n in self.variables_names], dtype=int) indices = list(filter(lambda x: x not in evidence_indices, np.arange(Q.shape[0]))) new_indices = np.array(np.append(indices, evidence_indices), dtype=int) pos = np.where(np.array(indices) == idx)[0][0] _Q = (Q[new_indices, :])[:, new_indices] lim_a = np.size(indices) Qaa = _Q[:lim_a, :lim_a] Qab = _Q[:lim_a, lim_a:] iQaa = inv(Qaa) mean_a = self.mean_[indices] mean_b = self.mean_[evidence_indices] std = self.nodes[name][1] mean = mean_a - (np.dot(iQaa, np.dot(Qab, (data[:, evidence_indices] - mean_b).T))).T ll = np.sum(norm.logpdf(data[:, idx], mean[:, pos], std)) return ll
def _get_responsibilities(self, pi, g, beta, mu_ivp, alpha): """ Gets the posterior responsibilities for each comp. of the mixture. """ probs = [[]]*len(self.N_data) for i, ifx in enumerate(self._ifix): zM = self._forward(g, beta, mu_ivp[i], ifx) for q, yq in enumerate(self.Y_train_): logprob = norm.logpdf( yq, zM[self.data_inds[q], :, q], scale=1/np.sqrt(alpha)) # sum over the dimension component logprob = logprob.sum(-1) if probs[q] == []: probs[q] = logprob else: probs[q] = np.column_stack((probs[q], logprob)) probs = [lp - pi for lp in probs] # subtract the maxmium for exponential normalize probs = [p - np.atleast_1d(p.max(axis=-1))[:, None] for p in probs] probs = [np.exp(p) / np.exp(p).sum(-1)[:, None] for p in probs] return probs
def mll(y_true, y_pred, y_var): """ Mean log loss under a Gaussian distribution. Parameters ---------- y_true: ndarray vector of true targets y_pred: ndarray vector of predicted targets y_var: float or ndarray predicted variances Returns ------- float: The mean negative log loss (negative log likelihood) Example ------- >>> y_true = np.random.randn(100) >>> mean_prob = - norm.logpdf(1e-2, loc=0) # -ve log prob close to mean >>> mll(y_true, y_true, 1) <= mean_prob # Should be good predictor True >>> mll(y_true, np.random.randn(100), 1) >= mean_prob # naive predictor True """ return - norm.logpdf(y_true, loc=y_pred, scale=np.sqrt(y_var)).mean()
def log_prior(self, theta): """ Returns the (log) prior probability of parameters theta. TODO NOTE: The returned quantity is an improper prior as its integral over the parameter space is not equal to 1. Parameters ---------- theta : array_like An array giving the autocorrelation parameter(s). Returns ------- log_p : float The (log) prior probability of parameters theta. An improper probability. """ theta_gp, theta_nn = self._parse_theta(theta) if self.prior_nn_scale == np.inf: prior_nn = 0.0 else: prior_nn = norm.logpdf(theta_nn, scale=self.prior_nn_scale).sum() if self.prior_gp_scale == np.inf: prior_gp = 0.0 else: prior_gp = expon.logpdf(theta_gp, scale=self.prior_gp_scale).sum() return prior_nn + prior_gp
def likelihood(param): ROOF = param[0] WALL = param[1] WIN = param[2] SHGC = param[3] EPD = param[4] LPD = param[5] HSP = param[6] CSP = param[7] OCC = param[8] INF = param[9] Boiler = param[10] COP = param[11] """ #--------------------------------- #to do: get the EUI from energyplus prediction = -0.2177929 - 0.2458677 * ROOF - 0.3842544 * WALL - 0.0049753 * WIN \ + 0.0176093 * SHGC + 0.014322 * EPD + 0.0125891 * LPD + 0.0518334 * HSP \ +0.0026691 * CSP + 0.0003318 * OCC + 0.5664563* INF - 0.6163375 * Boiler \ - 0.0425599 * COP #--------------------------------- """ prediction = param[12] # To read prediction value from run_metropolis_MCMC function singlelikelihoods = norm.logpdf(x=y, loc=prediction, scale=sd) #sumll = sum(singlelikelihoods) return singlelikelihoods
def uncollapsed_likelihood(self, X, parameters): """ Calculates the score of the data X under this component model with mean mu and precision rho. Inputs: X: A column of data (numpy) parameters: a dict with the following keys mu: the Gaussian mean rho: the precision of the Gaussian """ check_data_type_column_data(X) check_model_params_dict(parameters) mu = parameters['mu'] rho = parameters['rho'] N = float(len(X)) hypers = self.get_hypers() s = hypers['s'] r = hypers['r'] nu = hypers['nu'] m = hypers['mu'] sum_err = numpy.sum((mu-X)**2.0) log_likelihood = self.log_likelihood(X, {'mu':mu, 'rho':rho}) log_prior_mu = norm.logpdf(m, (r/rho)**.5) log_prior_rho = -(nu/2.0)*LOG_2+(nu/2.0)*math.log(s)+ \ (nu/2.0-1.0)*math.log(rho)-.5*s*rho-math.lgamma(nu/2.0) log_p = log_likelihood + log_prior_mu + log_prior_rho return log_p
def __init__(self, K, Y, init=None, threshold=1e-9): N = np.shape(K)[0] f = np.zeros((N,1)) converged = False k = 0 innerC = 0 for i in xrange(N): pdfDiff = norm.logpdf(f) - norm.logcdf(Y*f) W = np.exp(2*pdfDiff) + Y*f*np.exp(pdfDiff) Wsqrt = np.sqrt(W) Wdiag= np.diag(Wsqrt.flatten()) B = np.identity(N) + np.dot(Wdiag, np.dot(K, Wdiag)) grad = Y*np.exp(pdfDiff) b = W*f + grad interim = np.dot(Wdiag, np.dot(K, b)) cgRes = Cg(B, interim, threshold=threshold) s1 = cgRes.result innerC = innerC + cgRes.iterations a = b - Wsqrt*s1 if(converged): break f_prev = f f = np.dot(K, a) diff = f - f_prev if (np.dot(diff.T,diff).flatten() < threshold*N or innerC>15000): converged = True k = k+1 self.result = f self.iterations = k + innerC
def _logp_w(I, V, W, Y, mu_W, sigma_W): # Indices and lengths idx_done = I == 2 idx_traded_away = I == 1 idx_not_traded = I == 0 _, l = W.shape # Done case: Y < min(V, W) => min(W) > Y W_lower = (Y[idx_done] - mu_W) / sigma_W logp_done = truncnorm_logpdf( W[idx_done], W_lower, np.inf, loc=mu_W, scale=sigma_W) # Traded away case: min(W) < min(Y, V) = m (Working) m = np.minimum(Y[idx_traded_away], V[idx_traded_away]) F_0_m = 1 - (1 - norm.cdf(m, mu_W, sigma_W)) logp_traded_away = np.sum(np.log(1. / F_0_m * l) + norm.logpdf(W[idx_traded_away, 0], loc=mu_W, scale=sigma_W) + np.log((1 - norm.cdf(W[idx_traded_away, 0], loc=mu_W, scale=sigma_W))**(l-1))) # Not traded case: min(W) > V (and Y > V) W_lower = (V[idx_not_traded] - mu_W) / sigma_W logp_not_traded = truncnorm_logpdf( W[idx_not_traded], W_lower, np.inf, loc=mu_W, scale=sigma_W) return logp_done + logp_traded_away + logp_not_traded
def loglikelihood(self, g, beta, mu_ivp, alpha, pi, priors): logprobs = [] for i, ifx in enumerate(self._ifix): # get the logprobability for each mixture component ll = 0. zM = self._forward(g, beta, mu_ivp[i], ifx) for q, yq in enumerate(self.Y_train_): ll += norm.logpdf( yq, zM[..., q], scale=1/np.sqrt(alpha)).sum() logprobs.append(ll + np.log(pi[i])) logprobs = np.array(logprobs) lpmax = max(logprobs) loglik = lpmax + np.log(np.exp(logprobs - lpmax).sum()) Cg = self.latentforces[0].kernel(self.ttc[:, None]) Cg[np.diag_indices_from(Cg)] += 1e-5 Lg = np.linalg.cholesky(Cg) logprior = -0.5 * g.dot(cho_solve((Lg, True), g)) - \ np.log(np.diag(Lg)).sum() - \ Lg.shape[0] / 2 * np.log(2 * np.pi) for vn, x in zip(['beta'], beta): try: prior_logpdf = priors[vn] logprior += prior_logpdf(x) except KeyError: pass return loglik + logprior
def log_liklihood(logL0, a,b, B_l, sigma,M, z, logRich): p = 0 #mass function p+= np.sum(log_n_approx(M,z))#not normalized, if that's a problem i can approximate it. #lillihood of richness p+=np.sum(norm.logpdf(logRich, loc =logLam(logL0, a, b, B_l, M, z), scale = sigma)) return p
def dnorm(x, mean=0, sd=1, log=False): """ ============================================================================ dnorm() ============================================================================ Density Function for the normal distribution. Returns the probability density value at the value x. USAGE: cnorm(mean=0, sd=1, type="equal", conf=0.95) dnorm(x, mean=0, sd=1, log=False) pnorm(q, mean=0, sd=1, lowertail=True, log=False) qnorm(p, mean=0, sd=1, lowertail=True, log=False) rnorm(n=1, mean=0, sd=1) :param x (float, array of floats): The value(s) of x :param mean (float): mean of the distribution :param sd (float): standard deviation :param log (bool): take the log? :return: returns an array of density values ============================================================================ """ if log: return norm.logpdf(x, loc=mean, scale=sd) else: return norm.pdf(x, loc=mean, scale=sd)
def log_likelihood(self, g, beta, mu_ivp, alpha_prec, pi, ifix): # Has to be a nicer way of doing this using slices inds = [np.concatenate([self.data_inds[q] + k*self.dim.N for k in range(self.dim.K)]) for q in range(len(self.y_train_))] logps = [] for m, ifx in enumerate(ifix): Km = self._K(g, beta, ifx) means = np.kron(mu_ivp[:, m, :], np.ones(self.dim.N)).T L = np.linalg.matrix_power(Km, self.order) L = L.dot(means) # sum over the experiments logp_m = 0. for q, y in enumerate(self.y_train_): inds_q = inds[q] Lim_q = L[inds_q, q] eta = Lim_q - y logp_m += norm.logpdf(eta, scale=1/np.sqrt(alpha_prec)).sum() logps.append(logp_m + np.log(pi[m])) logps = np.array(logps) a = logps.max() res = a + np.log(np.exp(logps - a).sum()) return res
def test_KL_divergence_for_normal_distributions(show_plot=True): mu_0 = 0 sigma_0 = 1 interval = norm.interval(.99,mu_0,sigma_0) support = numpy.linspace(interval[0], interval[1], num=2000) mus = numpy.linspace(0, 3, num=30) p_0 = norm.logpdf(support, mu_0, sigma_0) KL_inf = [] KL_ana = [] for mu in mus: p_1 = norm.logpdf(support, mu, sigma_0) kld = qtu.KL_divergence_arrays(support, p_0, p_1, False) KL_inf.append(float(kld)) KL_ana.append(actual_KL(mu_0, sigma_0, mu, sigma_0)) KL_inf = numpy.array(KL_inf) KL_ana = numpy.array(KL_ana) KL_diff = KL_ana-KL_inf if show_plot: pylab.subplot(1,2,1) pylab.plot(KL_inf, label='est') pylab.plot(KL_ana, label='analytical') pylab.title('estimated KL') pylab.legend() pylab.subplot(1,2,2) pylab.plot(KL_diff) pylab.title('KL error') pylab.show() _, p = pearsonr(KL_inf, KL_ana) return p
def loglikelihood(X, Z, W): ZW = Z.dot(W.T) LL = 0 for i in xrange(N): ll = mvn.logpdf(X[i], mean=ZW[i], cov=sigmaI) LL += ll LL += norm.logpdf(W.flatten(), scale=1/lam).sum() return LL
def test_aic_0(): d = norm.rvs(size=1000) p = norm.logpdf(d) c = ChainConsumer() c.add_chain(d, posterior=p, num_free_params=1, num_eff_data_points=1000) aics = c.comparison.aic() assert len(aics) == 1 assert aics[0] == 0
def test_bic_fail_no_data_points(): d = norm.rvs(size=1000) p = norm.logpdf(d) c = ChainConsumer() c.add_chain(d, posterior=p, num_free_params=1) bics = c.comparison.bic() assert len(bics) == 1 assert bics[0] is None
def log_liklihood(logL0, a,b, B_l, sigma, z, logRich): logPRich = np.array([norm.logpdf(lr, loc =logLam(logL0, a, b, B_l, z, ms), scale = sigma)\ for lr, ms in izip(logRich, massSamples)]) logL_k = logsumexp(logPRich+logPMass-logPSample, axis = 1) - np.log(nSamples)#mean of weights return np.sum(logL_k)
def log_p_MWL(S, I): mu = I / log_p_MWLLen sigma = 0.005 # # log of normal density, p(S|R) # The probability density function for norm is: # norm.pdf(x) = exp(-x**2/2)/sqrt(2*pi) return norm.logpdf(np.float(S), loc=np.float(mu), scale=np.float(sigma))
def test_dic_0(): d = norm.rvs(size=1000) p = norm.logpdf(d) c = ChainConsumer() c.add_chain(d, posterior=p) dics = c.comparison.dic() assert len(dics) == 1 assert dics[0] == 0
def metropolis_bivariate(y1, y2, start_theta1, start_theta2, num_samples, include_start=True, include_loglik=False): if include_start: theta1_samples = [start_theta1] theta2_samples = [start_theta2] else: theta1_samples = [] theta2_samples = [] loglik_samples = [] current_theta1 = start_theta1 current_theta2 = start_theta2 # somehow the book is using unnormalized log probability. don't know why. current_log_prob = norm.logpdf((current_theta1,current_theta2),loc=(0,0),scale=(1,1)).sum() \ - norm.logpdf((0,0),loc=(0,0),scale=(1,1)).sum() for i in xrange(num_samples): proposal_theta1, proposal_theta2 = np.random.normal(loc=(current_theta1, current_theta2), scale=(0.2,0.2)) proposal_log_prob = norm.logpdf((proposal_theta1,proposal_theta2), loc=(0,0),scale=(1,1)).sum() \ - norm.logpdf((0,0), loc=(0,0),scale=(1,1)).sum() if proposal_log_prob > current_log_prob: flag_accept = True else: acceptance_prob = np.exp(proposal_log_prob - current_log_prob) if np.random.random() < acceptance_prob: flag_accept = True else: flag_accept = False if flag_accept: current_theta1 = proposal_theta1 current_theta2 = proposal_theta2 current_log_prob = proposal_log_prob theta1_samples.append(current_theta1) theta2_samples.append(current_theta2) loglik_samples.append(current_log_prob) if include_loglik: return theta1_samples, theta2_samples, loglik_samples else: return theta1_samples, theta2_samples
def test_one_dim(self): """ Test log pdf for one dimensional data """ x = np.array([1]) mean = np.array([2]) Q = np.array([[ 1 / 25 ]]) gmrf = GMRF() self.assertAlmostEqual(gmrf._logpdf(x, mean, Q), norm.logpdf(1, 2, 5))
def _compute_ev_det_loglike(physics, event, det): # load the station tuple and compute basic event-station attributes like # distance, travel time, azimuth, and azimuth difference stanum = det.stanum station = STATIONS[stanum] dist = compute_distance((station.lon, station.lat), (event.lon, event.lat)) ttime = compute_travel_time(dist) sta_to_ev_az = compute_azimuth((station.lon, station.lat), (event.lon, event.lat)) # the azimuth difference of observed to theoretical degdiff = compute_degdiff(sta_to_ev_az, det.azimuth) loglike = 0 # detection probability detprob = logistic(physics.mu_d0[stanum] + physics.mu_d1[stanum] * event.mag + physics.mu_d2[stanum] * dist) loglike += log(detprob) # detection time loglike += laplace.logpdf(det.time, event.time + ttime + physics.mu_t[stanum], physics.theta_t[stanum]) # detection azimuth loglike += laplace.logpdf(degdiff, physics.mu_z[stanum], physics.theta_z[stanum]) # detection slowness loglike += laplace.logpdf(det.slowness, compute_slowness(dist) + physics.mu_s[stanum], physics.theta_s[stanum]) # detection amplitude loglike += norm.logpdf(log(det.amp), physics.mu_a0[stanum] + physics.mu_a1[stanum] * event.mag + physics.mu_a2[stanum] * dist, physics.sigma_a[stanum]) return loglike
def _loglikelihood(self, v): coef, sigma = v[:-1], v[-1] y_hat = np.dot(coef, self.X_.T) return np.sum(norm.logpdf(self.y_, y_hat, sigma))
.. figure:: ../../examples/resources/table_comparison.png :align: center :width: 60% """ ############################################################################### # The code to produce this, and the raw LaTeX, is given below: from scipy.stats import norm from chainconsumer import ChainConsumer n = 10000 d1 = norm.rvs(size=n) p1 = norm.logpdf(d1) p2 = norm.logpdf(d1, scale=1.1) c = ChainConsumer() c.add_chain(d1, posterior=p1, name="Model A", num_eff_data_points=n, num_free_params=4) c.add_chain(d1, posterior=p2, name="Model B", num_eff_data_points=n, num_free_params=5) c.add_chain(d1, posterior=p2,
def negative_log_proba(y_true, y_pred, s_pred): nlp = -norm.logpdf(y_true, loc=y_pred, scale=s_pred) mean_nlp = np.mean(nlp) return mean_nlp
def T14(pv): pv = atleast_2d(pv) a = as_from_rhop(pv[:, 2], pv[:, 1]) t14 = duration_eccentric(pv[:, 1], sqrt(pv[:, 4]), a, arccos(pv[:, 3] / a), 0, 0, 1) return norm.logpdf(t14, mean, std)
def train_and_evaluate(self, x_train: np.ndarray, y_train: np.ndarray, x_valid: np.ndarray, y_valid: np.ndarray, num_steps: int = 13000, validate_every_n_steps=1000, keep_every: int = 100, num_burn_in_steps: int = 3000, lr: float = 1e-2, epsilon: float = 1e-10, batch_size: int = 20, mdecay: float = 0.05, verbose=False): """ Train and validates the neural network :param x_train: input training datapoints. :param y_train: input training targets. :param x_valid: validation data points :param y_valid: valdiation targets :param num_steps: Number of sampling steps to perform after burn-in is finished. In total, `num_steps // keep_every` network weights will be sampled. :param validate_every_n_steps: :param keep_every: Number of sampling steps (after burn-in) to perform before keeping a sample. In total, `num_steps // keep_every` network weights will be sampled. :param num_burn_in_steps: Number of burn-in steps to perform. This value is passed to the given `optimizer` if it supports special burn-in specific behavior. Networks sampled during burn-in are discarded. :param lr: learning rate :param batch_size: batch size :param epsilon: epsilon for numerical stability :param mdecay: momemtum decay :param verbose: verbose output """ assert batch_size >= 1, "Invalid batch size. Batches must contain at least a single sample." if x_train.shape[0] < batch_size: logging.warning( "Not enough datapoints to form a batch. Use all datapoints in each batch" ) batch_size = x_train.shape[0] # burn-in self.train(x_train, y_train, num_burn_in_steps=num_burn_in_steps, num_steps=num_burn_in_steps, lr=lr, epsilon=epsilon, mdecay=mdecay, verbose=verbose) learning_curve_mse = [] learning_curve_ll = [] n_steps = [] for i in range(num_steps // validate_every_n_steps): self.train(x_train, y_train, num_burn_in_steps=0, num_steps=validate_every_n_steps, lr=lr, epsilon=epsilon, mdecay=mdecay, verbose=verbose, keep_every=keep_every, continue_training=True, batch_size=batch_size) mu, var = self.predict(x_valid) ll = np.mean(norm.logpdf(y_valid, loc=mu, scale=np.sqrt(var))) mse = np.mean((y_valid - mu)**2) step = num_burn_in_steps + (i + 1) * validate_every_n_steps learning_curve_ll.append(ll) learning_curve_mse.append(mse) n_steps.append(step) if verbose: print("Validate : NLL = {:11.4e} MSE = {:.4e}".format( -ll, mse)) return n_steps, learning_curve_ll, learning_curve_mse
def likelihood(a, b, sd): pred = a * x + b # predictions single_likelihood = norm.logpdf(y, loc=pred, scale=sd) # loc=mean, scale=var return sum(single_likelihood)
def as_prior(pv): a = as_from_rhop(pv[2], pv[1]) return norm.logpdf(a, mean, std)
def sample_and_plot(sess, kld, kl_from_pq, kl_from_cob, p_samples, q_samples, m_samples, log_ratio_p_q, log_ratio_p_m, mu_1, mu_2, scale_p, scale_q, mu_3, scale_m): kl_ratio_store=[] log_ratio_store=[] log_r_p_from_m_direct_store=[] feed_dict = {} kl_ratio, kl_true, kl_cob, p_s, q_s, m_s, lpq, lpq_from_cob_dre_direct = sess.run([kld, kl_from_pq, kl_from_cob, p_samples, q_samples, m_samples, log_ratio_p_q, log_ratio_p_m], feed_dict=feed_dict) '''Save ratio estimates''' data_dir = "../data/sym/"+str(scale_p)+"-"+str(scale_q)+str(scale_m)+"/" if not os.path.exists(data_dir): os.makedirs(data_dir) f = open(data_dir+"KLD"+".txt", "a") f.write("GT for mu_3 = "+str(mu_3)+": "+str(kl_ratio)+"\nGT-est: "+str(kl_true)+"\nCoB: "+str(kl_cob)+"\n----------\n") f.close() log_ratio_store.append(lpq) log_r_p_from_m_direct_store.append(lpq_from_cob_dre_direct) pickle.dump(log_r_p_from_m_direct_store, open(data_dir+"log_r_p_from_m_direct_store"+str(mu_3)+".p", "wb")) pickle.dump(m_s, open(data_dir+"xs"+str(mu_3)+".p", "wb")) pickle.dump(log_ratio_store, open(data_dir+"log_ratio_store"+str(mu_3)+".p", "wb")) xs = m_s fig, [ax1,ax2,ax3, ax4] = plt.subplots(1, 4,figsize=(13,4)) ax1.hist(p_s, density=True, histtype='stepfilled', alpha=0.8, label='P') ax1.hist(q_s, density=True, histtype='stepfilled', alpha=0.8, label='Q') ax1.hist(m_s, density=True, histtype='stepfilled', alpha=0.8, label='M') ax1.legend(loc='best', frameon=False) ax1.set_xlim([-5,5]) ax2.scatter(xs,log_ratio_store[0],label='True p/q',alpha=0.9,s=10.,c='b') ax2.scatter(xs,log_r_p_from_m_direct_store[-1][:,0]-log_r_p_from_m_direct_store[-1][:,1],label='CoB p/q',alpha=0.9,s=10.,c='r') ax2.scatter(xs,-log_ratio_store[0],label='True q/p',alpha=0.9,s=10.,c='b') ax2.scatter(xs,log_r_p_from_m_direct_store[-1][:,1]-log_r_p_from_m_direct_store[-1][:,0],label='CoB q/p',alpha=0.9,s=10.,c='r') ax2.set_xlabel("Samples") ax2.set_ylabel("Log Ratio") ax2.legend(loc='best') ax2.set_xlim([-6,10]) ax2.set_ylim([-1000,1000]) pm = [np.squeeze(norm.logpdf(x,mu_1,scale_p)-cauchy.logpdf(x,mu_3,scale_m)) for x in xs] qm = [np.squeeze(norm.logpdf(x,mu_2,scale_q)-cauchy.logpdf(x,mu_3,scale_m)) for x in xs] ax4.scatter(xs,pm,label='True p/m',alpha=0.9,s=10.,c='b') ax4.scatter(xs,log_r_p_from_m_direct_store[-1][:,0]-log_r_p_from_m_direct_store[-1][:,2],label='CoB p/m',alpha=0.9,s=10.,c='r') ax4.scatter(xs,qm,label='True q/m',alpha=0.9,s=10.,c='y') ax4.scatter(xs,log_r_p_from_m_direct_store[-1][:,1]-log_r_p_from_m_direct_store[-1][:,2],label='CoB q/m',alpha=0.9,s=10.,c='g') ax4.set_xlabel("Samples") ax4.set_ylabel("Log Ratio") ax4.legend(loc='best') ax4.set_xlim([-6,10]) ax4.set_ylim([-1000,1000]) rat = log_r_p_from_m_direct_store[-1][:,0]-log_r_p_from_m_direct_store[-1][:,1] d = [np.squeeze(norm.logpdf(x,mu_2,scale_q)) for x in xs] b = [np.squeeze(norm.logpdf(x,mu_1,scale_p)) for x in xs] ax3.scatter(xs,b,label='True P',alpha=0.9,s=5.) ax3.scatter(xs,rat+d,label='P',alpha=0.9,s=5.) ax3.set_xlabel("Samples") ax3.set_ylabel("Log P(x)") ax3.legend(loc='best') ax3.set_xlim([-6,10]) ax3.set_ylim([-600,400]) plt.savefig(data_dir+str(mu_3)+".jpg")
def main(old_samples_tsv, old_segmentations_tsv, new_samples_tsv, benchmark=True): """ Infer and print the new values for mu and sigma (for each of S, L, A, P, C, T) to STDOUT. Args: * old_samples_tsv: path to TSV file containing polya-samples data from an older kmer model. * old_segmentations_tsv: path to TSV file containing polya-segmentation data from an older kmer model. * new_samples_tsv: path to TSV file containing polya-samples data from the newer kmer model. Returns: N/A, prints outputs to STDOUT. """ ### read all samples into numpy arrays: print("Loading data from TSV...") old_data = old_tsv_to_numpy(old_samples_tsv) segmentations = make_segmentation_dict(old_segmentations_tsv) new_data = new_tsv_to_numpy(new_samples_tsv, segmentations) print("... Datasets loaded.") ### infer best possible new mu,sigma for each of S, L, A, P, T: print("Fitting gaussians to new scaled samples (this may take a while)...") new_mu_S, new_sigma_S = fit_gaussian(new_data['S_samples']) new_mu_L, new_sigma_L = fit_gaussian(new_data['L_samples']) (new_pi0_A, new_mu0_A, new_sig0_A), (new_pi1_A, new_mu1_A, new_sig1_A) = fit_gmm(new_data['A_samples'], ncomponents=2) new_mu_P, new_sigma_P = fit_gaussian(new_data['P_samples']) (new_pi0_T, new_mu0_T, new_sig0_T), (new_pi1_T, new_mu1_T, new_sig1_T) = fit_gmm(new_data['T_samples'], ncomponents=2) ### print to stdout: print("New params for START: mu = {0}, var = {1}, stdv = {2}".format( new_mu_S, new_sigma_S, np.sqrt(new_sigma_S))) print("New params for LEADER: mu = {0}, var = {1}, stdv = {2}".format( new_mu_L, new_sigma_L, np.sqrt(new_sigma_L))) print("New params for ADAPTER0: pi = {0}, mu = {1}, var = {2}, stdv = {3}". format(new_pi0_A, new_mu0_A, new_sig0_A, np.sqrt(new_sig0_A))) print("New params for ADAPTER1: pi = {0}, mu = {1}, var = {2}, stdv = {3}". format(new_pi1_A, new_mu1_A, new_sig1_A, np.sqrt(new_sig1_A))) print("New params for POLYA: mu = {0}, var = {1}, stdv = {2}".format( new_mu_P, new_sigma_P, np.sqrt(new_sigma_P))) print("New params for TRANSCR0: pi = {0}, mu = {1}, var = {2}, stdv = {3}". format(new_pi0_T, new_mu0_T, new_sig0_T, np.sqrt(new_sig0_T))) print("New params for TRANSCR1: pi = {0}, mu = {1}, var = {2}, stdv = {3}". format(new_pi1_T, new_mu1_T, new_sig1_T, np.sqrt(new_sig1_T))) ### optionally, benchmark: if not benchmark: return print("===== Emission Log-Likelihood Benchmarks =====") old_S_llh = np.mean(old_data['S_loglkhd']) new_S_llh = np.mean( norm.logpdf(new_data['S_samples'], loc=new_mu_S, scale=np.sqrt(new_sigma_S))) print("> Average START log-probs:") print( "> Old avg. log-likelihood: {0} | New avg. log-likelihood: {1}".format( old_S_llh, new_S_llh)) old_L_llh = np.mean(old_data['L_loglkhd']) new_L_llh = np.mean( norm.logpdf(new_data['L_samples'], loc=new_mu_L, scale=np.sqrt(new_sigma_L))) print("> Average LEADER log-probs:") print( "> Old avg. log-likelihood: {0} | New avg. log-likelihood: {1}".format( old_L_llh, new_L_llh)) old_A_llh = np.mean(old_data['A_loglkhd']) new_A_llh0 = new_pi0_A * norm.pdf( new_data['A_samples'], loc=new_mu0_A, scale=np.sqrt(new_sig0_A)) new_A_llh1 = new_pi1_A * norm.pdf( new_data['A_samples'], loc=new_mu1_A, scale=np.sqrt(new_sig1_A)) new_A_llh = np.mean(np.log(new_A_llh0 + new_A_llh1)) print("> Average ADAPTER log-probs:") print( "> Old avg. log-likelihood: {0} | New avg. log-likelihood: {1}".format( old_A_llh, new_A_llh)) old_P_llh = np.mean(old_data['P_loglkhd']) new_P_llh = np.mean( norm.logpdf(new_data['P_samples'], loc=new_mu_P, scale=np.sqrt(new_sigma_P))) print("> Average POLYA log-probs:") print( "> Old avg. log-likelihood: {0} | New avg. log-likelihood: {1}".format( old_P_llh, new_P_llh)) old_T_llh = np.mean(old_data['T_loglkhd']) new_T_llh0 = new_pi0_T * norm.pdf( new_data['T_samples'], loc=new_mu0_T, scale=np.sqrt(new_sig0_T)) new_T_llh1 = new_pi1_T * norm.pdf( new_data['T_samples'], loc=new_mu1_T, scale=np.sqrt(new_sig1_T)) new_T_llh = np.mean(np.log(new_T_llh0 + new_T_llh1)) print("> Average TRANSCRIPT log-probs:") print( "> Old avg. log-likelihood: {0} | New avg. log-likelihood: {1}".format( old_T_llh, new_T_llh))
def __call__(self, new_params, old_params, old_grad_params): return np.sum(norm.logpdf(new_params, loc=(old_params + 0.5 * self.dt ** 2 * old_grad_params), scale=self.dt))
def q_log_pdf(θold, θ): # bit of a clusterfuck here about subtracting dataframes as the indicies are unordered logpdf = norm.logpdf((θold.values - θ.values) / scale_walk, loc=0, scale=1) return np.sum(logpdf, axis=1)
dPSIL = hour2day(PSIL_hat, idx)[~discard_vod] VOD_hat, popt = fitVOD_RMSE(dPSIL, dLAI, VOD_ma, return_popt=True) dS1 = hour2day(S1_hat, idx)[~discard_vod][::2] if np.isfinite(np.nansum(dS1)) and np.nansum(dS1) > 0: counts, bin_edges = np.histogram(dS1, bins=bins, normed=True) cdf2 = np.cumsum(counts) / sum(counts) dS1_matched = np.array([ bin_edges[np.abs(cdf1 - cdf2[int(itm * 100)]).argmin()] for itm in dS1 ]) else: dS1_matched = np.zeros(dS1.shape) + np.nan loglik_vod = np.nanmean( norm.logpdf(VOD_ma, VOD_hat, theta[idx_sigma_vod])) loglik_et = np.nanmean(norm.logpdf(ET, ET_hat, theta[idx_sigma_et])) loglik_sm = np.nanmean( norm.logpdf(SOILM, dS1_matched, theta[idx_sigma_sm])) TS = [ np.concatenate([TS[ii], itm]) for ii, itm in enumerate((VOD_hat, ET_hat, PSIL_hat, dS1_matched)) ] PARA = [ np.concatenate([PARA[ii], itm]) for ii, itm in enumerate(( popt, theta, np.array([loglik_vod, loglik_et, loglik_sm]))) ] TS = [np.reshape(itm, [nsample, -1]) for itm in TS] # VOD,ET,PSIL,S1 PARA = [np.reshape(itm, [nsample, -1]) for itm in PARA]
def edge_cost(self, edge: GraphEdge) -> float: rule_id = self.rule_set.get_id(edge.rule) return -norm.logpdf(edge.source.logfreq - edge.target.logfreq, self.means[rule_id, ], self.sdevs[rule_id, ])
def calc_predictive_logp(x, mu, sigma, l, h): return norm.logpdf(x, loc=mu, scale=sigma)
def gaussian_new(obs, x, params): x_vec = np.sqrt((params.beta**2) * np.exp(x)) return norm.logpdf(obs, loc=0, scale=x_vec)
# Gaussian distribution (normal distribution ) is a very common continuous probability distribution # scipy is a fast way to calculate from scipy.stats import norm import numpy as np # probability density of a zero from standart normal norm.pdf(0) # if mean = 5 and standart devation = 10 norm.pdf(0, loc=5, scale=10) # to calculate the probability densities of many diff-t values # 1st) create an array r = np.random.randn(10) # 2nd) calculate the pdf of all values stored in array at the same time norm.pdf(r) # we can calculate lognorm norm.logpdf(r) # cumulative distribution function (CDF) # this is the integral of the PDAF from -inf to x norm.cdf(r) # and also log CDF norm.logcdf(r)
#! python3 from scipy.stats import norm import numpy as np # Gaussian Probability Density Function # default mean = 0 and variance = 1 print(norm.pdf(0)) # Gaussian with mean == loc = 5 and standard deviation == scale = 10 == variance = 100 print(norm.pdf(0, loc=5, scale=10)) r = np.random.randn(10) print(norm.pdf(r)) # Joint log probability print(norm.logpdf(r)) # Cumulative Distribution function == integral of pdf from -infinity to x print(norm.cdf(r)) print(norm.logcdf(r))
def model(parm, h, self): """ Calculate the log probability of a model `h` [instance of :class:`hmf._framework.Framework`] with parameters ``parm``. At the moment, this is a little hacky, because the parameters have to be the first argument (for both Minimize and MCMC), so we use a function and pass self last. Parameters ---------- parm : list of floats The position of the model. Takes arbitrary parameters. h : instance of :class:`~_framework.Framework` An instance of any subclass of :class:`~_framework.Framework` with the desired options set. Variables of the estimation are updated within the routine. Returns ------- ll : float The log likelihood of the model at the given position. """ if self.verbose > 1: print(("Params: ", list(zip(self.attrs, parm)))) ll = 0 p = copy.copy(parm) for prior in self.priors: if type(prior.name) == list: index = [self.attrs.index(name) for name in prior.name] else: index = self.attrs.index(prior.name) ll += prior.ll(parm[index]) if np.isinf(ll): return ret_arg(ll, self.blobs) # If it is a log distribution, un-log it for use. if isinstance(prior, Log): p[index] = 10**parm[index] # Rebuild the hod dict from given vals # Any attr starting with <name>: is put into a dictionary. param_dict = {} for attr, val in zip(self.attrs, p): if ":" in attr: if attr.split(":")[0] not in param_dict: param_dict[attr.split(":")[0]] = {} param_dict[attr.split(":")[0]][attr.split(":")[1]] = val else: param_dict[attr] = val # Update the actual model try: # This try: except: should capture poor parameter choices quickly. h.update(**param_dict) except ValueError as e: if self.relax: print(("WARNING: PARAMETERS FAILED ON UPDATE, RETURNING INF: ", list(zip(self.attrs, parm)))) print(e) print((traceback.format_exc())) return ret_arg(-np.inf, self.blobs) else: print((traceback.format_exc())) raise e # Get the quantity to compare (if exceptions are raised, treat properly) try: q = getattr(h, self.quantity) except Exception as e: if self.relax: print(( "WARNING: PARAMETERS FAILED WHEN CALCULATING QUANTITY, RETURNING INF: ", list(zip(self.attrs, parm)))) print(e) print((traceback.format_exc())) return ret_arg(-np.inf, self.blobs) else: print((traceback.format_exc())) raise e # The logprob of the model if self.cov: ll += _lognormpdf(q, self.data, self.sigma) else: ll += np.sum(norm.logpdf(self.data, loc=q, scale=self.sigma)) # Add the likelihood of the contraints for k, v in list(self.constraints.items()): ll += norm.logpdf(getattr(h, k), loc=v[0], scale=v[1]) if self.verbose > 2: print(("CONSTRAINT: ", k, getattr(h, k))) if self.verbose: print(("Likelihood: ", ll)) if self.verbose > 1: print(("Update Dictionary: ", param_dict)) if self.verbose > 2: print(("Final Quantity: ", q)) # Get blobs to return as well. if self.blobs is not None: out = [] for b in self.blobs: if ":" not in b: out.append(getattr(h, b)) elif ":" in b: out.append(getattr(h, b.split(":")[0])[b.split(":")[1]]) return ll, out else: return ll
def log_likelihood(self, x): log_l = np.zeros(x.size) for n in self.names: log_l += norm.logpdf(x[n]) return log_l
def forward_filter(self): T = self.T # Number of timesteps obs_dim = self.obs_dim # Dimension of observed data state_dim = self.state_dim # Dimension of state vector if self.obs_discount: self.gamma_n = np.zeros(T) self.s = np.zeros(T) self.s[0] = self.s0 else: V = self.V # Dimensions of [obs_dim,obs_dim] self.r = np.zeros(T) # For unknown obs. variance self.e = np.zeros([T,obs_dim]) # Forecast error self.f = np.zeros([T,obs_dim]) # Forecast mean self.m = np.zeros([T,state_dim]) # State vector/matrix posterior mean self.a = np.zeros([T,state_dim]) # State vector/matrix prior mean self.Q = np.zeros([T,obs_dim,obs_dim]) # Forecast covariance self.A = np.zeros([T,state_dim,obs_dim]) # Adaptive coefficient vector self.R = np.zeros([T,state_dim,state_dim]) # State vector prior variance self.C = np.zeros([T,state_dim,state_dim]) # State vector posterior variance self.B = np.zeros([T,state_dim,state_dim]) # Retrospective ??? # If we want to change the tracked quantities all at once later, # it would be handy to be able to reference all of them at the # same time. self.dynamic_names = ['F','Y','r' , 'e', 'f' ,'m' ,'a', 'Q', 'A', 'R','C','B'] if self.obs_discount: self.dynamic_names = self.dynamic_names + ['gamma_n','s'] if self.dynamic_G: self.dynamic_names = self.dynamic_names + ['G'] # Forward filtering # For each time step, we ingest a new observation and update our priors # to posteriors. for t in range(T): self.t = t self.filter_step(t) # The last thing we want to do is tabulate the current # step's contribution to the overall log-likelihood. if self.calculate_ll: if self.obs_discount: # We need the shape parameters for the preceding time step in the current # timestep's calculation of the log likelihood. This just offsets the # vector of shape parameters. shifted_gamma = np.roll(np.squeeze(self.gamma_n),1) shifted_gamma[0] = 1.0 self.log_likelihood = student_t.logpdf(np.squeeze(self.e), shifted_gamma, scale=np.squeeze(np.sqrt(self.Q))) else: self.log_likelihood = norm.logpdf(np.squeeze(self.e), scale=np.squeeze(np.sqrt(self.Q))) # This is the marginal model likelihood. self.ll_sum = np.sum(self.log_likelihood) if self.nancheck: try: for array in [self.A,self.C,self.Q,self.m,self.log_likelihood]: assert np.any(np.isnan(array)) == False except AssertionError: print 'NaN values encountered in forward filtering.' self.populate_scores() self.is_filtered = True
def logpdf(self, vals: np.ndarray) -> float: return norm.logpdf(vals, loc=self.mean, scale=self.variance**.5)
X = butter_lowpass_filter(X, cutoff=5, fs=32, order=6) Y = butter_lowpass_filter(Y, cutoff=5, fs=32, order=6) Z = butter_lowpass_filter(Z, cutoff=5, fs=32, order=6) #plotdata(original,filtered=Z,fs=32,t=T) D = np.column_stack((X, Y, Z)) X_norm = normalize(X, -2, 2) Y_norm = normalize(Y, -2, 2) Z_norm = normalize(Z, -2, 2) delta = norm.logpdf(Z_norm, loc=np.mean(Z_norm, dtype=np.float64, axis=0), scale=np.std(Z_norm, dtype=np.float64, axis=0)) delta3 = multivariate_normal.logpdf(D, mean=np.mean(D, dtype=np.float64, axis=0), cov=np.std(D, dtype=np.float64, axis=0)) XYZ_norm = np.column_stack((X_norm, Y_norm, Z_norm)) #xnp.savetxt('filtereddata/XYZ_norm1.txt',XYZ_norm,fmt='%1.15g') # for index in range(0,len(Z)): # if(delta[index]<-5): # print(index) # t = np.linspace(0, 2, 64, endpoint=False) # plt.plot(t, Z[index-32:index+32], 'g-', linewidth=2, label='filtered data') # plt.xlabel('Time [sec]')
def ll(self, param): return norm.logpdf(param, loc=self.mean, scale=self.sd)
def fwd_pass(w, x, y, z, var_Z): w = np.copy(w) tpts = w.shape[0] ntaxa = w.shape[1] A = self.A A_init = self.A_init # normalized forward probabilities alpha = np.zeros(w.shape) np.seterr(divide="ignore") # zeros are appropriately handled here p = np.concatenate((z[0] + var_Z, [0])) p = np.tile(p, ntaxa).reshape(ntaxa, ntaxa) w0 = np.tile(np.log(w[0]), ntaxa).reshape(ntaxa, ntaxa) np.fill_diagonal(w0, 0) p += w0 p = np.exp(p - logsumexp(p, axis=1, keepdims=True)) n = y[0].sum() y0 = np.tile(y[0], ntaxa).reshape(ntaxa, ntaxa) alpha_w1 = np.log(A_init) + multinomial(y0, n, p) alpha_w1[:ntaxa - 1] += norm.logpdf(loc=x[0], scale=np.sqrt(var_Z), x=z[0]) np.fill_diagonal(p, 0) p /= p.sum(axis=1, keepdims=True) alpha_w0 = np.zeros(ntaxa) alpha_w0[y[0] > 0] = -np.inf alpha_w0[y[0] == 0] = np.log(1 - A_init)[y[0] == 0] + multinomial( y0, y0.sum(), p)[y[0] == 0] alpha[0] = np.exp( alpha_w1 - logsumexp(np.vstack([alpha_w0, alpha_w1]).T, axis=1)) assert np.all(alpha[0] >= 0) and np.all(alpha[0] <= 1) for t in range(1, tpts): alpha_w1 = np.zeros(ntaxa) alpha_w0 = np.zeros(ntaxa) at0 = alpha[t - 1] p = np.concatenate((z[t] + var_Z, [0])) p = np.tile(p, ntaxa).reshape(ntaxa, ntaxa) w0 = np.tile(np.log(w[t]), ntaxa).reshape(ntaxa, ntaxa) np.fill_diagonal(w0, 0) p += w0 p = np.exp(p - logsumexp(p, axis=1, keepdims=True)) n = y[t].sum() y0 = np.tile(y[t], ntaxa).reshape(ntaxa, ntaxa) alpha_w1 = np.log(A[:, 1, 1] * at0 + A[:, 0, 1] * (1 - at0)) + multinomial(y0, n, p) alpha_w1[:ntaxa - 1] += norm.logpdf(loc=x[t], scale=np.sqrt(var_Z), x=z[t]) np.fill_diagonal(p, 0) p /= p.sum(axis=1, keepdims=True) alpha_w0 = np.zeros(ntaxa) alpha_w0[y[t] > 0] = -np.inf alpha_w0[y[t] == 0] = np.log(A[:, 1, 0] * at0 + A[:, 0, 0] * (1 - at0))[y[t] == 0] + multinomial( y0, n, p)[y[t] == 0] np.set_printoptions(threshold=np.inf) assert np.all( np.logical_or(np.isfinite(alpha_w0), np.isfinite(alpha_w1)) >= 1), str(p[0]) + "\n" + str(y[t]) alpha[t] = np.exp( alpha_w1 - logsumexp(np.vstack([alpha_w0, alpha_w1]).T, axis=1)) assert np.all(alpha[t] >= 0) and np.all( alpha[t] <= 1), alpha[t] return alpha
def logpdf_multiple(self, x): v = self.thresh(x[:, 0]) return norm.logpdf(x[:, 0], 0, self.sigma) + norm.logpdf( x[:, 1:], 0, np.sqrt(v)[:, None]).sum(1)
def compute(self, X, derivative=False, **kwargs): """ Computes the Log EI value and its derivatives. Parameters ---------- X: np.ndarray(1, D), The input point where the acquisition_functions function should be evaluate. The dimensionality of X is (N, D), with N as the number of points to evaluate at and D is the number of dimensions of one X. derivative: Boolean If is set to true also the derivative of the acquisition_functions function at X is returned Not implemented yet! Returns ------- np.ndarray(1,1) Log Expected Improvement of X np.ndarray(1,D) Derivative of Log Expected Improvement at X (only if derivative=True) """ if derivative: logger.error("LogEI does not support derivative \ calculation until now") return m, v = self.model.predict(X) _, eta = self.model.get_incumbent() f_min = eta - self.par s = np.sqrt(v) z = (f_min - m) / s log_ei = np.zeros([m.size]) for i in range(0, m.size): mu, sigma = m[i], s[i] # par_s = self.par * sigma # Degenerate case 1: first term vanishes if np.any(abs(f_min - mu) == 0): if sigma > 0: log_ei[i] = np.log(sigma) + norm.logpdf(z[i]) else: log_ei[i] = -np.Infinity # Degenerate case 2: second term vanishes and first term # has a special form. elif sigma == 0: if np.any(mu < f_min): log_ei[i] = np.log(f_min - mu) else: log_ei[i] = -np.Infinity # Normal case else: b = np.log(sigma) + norm.logpdf(z[i]) # log(y+z) is tricky, we distinguish two cases: if np.any(f_min > mu): # When y>0, z>0, we define a=ln(y), b=ln(z). # Then y+z = exp[ max(a,b) + ln(1 + exp(-|b-a|)) ], # and thus log(y+z) = max(a,b) + ln(1 + exp(-|b-a|)) a = np.log(f_min - mu) + norm.logcdf(z[i]) log_ei[i] = max(a, b) + np.log(1 + np.exp(-abs(b - a))) else: # When y<0, z>0, we define a=ln(-y), b=ln(z), # and it has to be true that b >= a in # order to satisfy y+z>=0. # Then y+z = exp[ b + ln(exp(b-a) -1) ], # and thus log(y+z) = a + ln(exp(b-a) -1) a = np.log(mu - f_min) + norm.logcdf(z[i]) if a >= b: # a>b can only happen due to numerical inaccuracies # or approximation errors log_ei[i] = -np.Infinity else: log_ei[i] = b + np.log(1 - np.exp(a - b)) return log_ei
def pairwise_pass(w, x, y, z, var_Z, alpha, gamma): w = np.copy(w) # w_{t-1}, w_t pairwise probabilities w0_w1 = np.zeros((w.shape[1], w.shape[0], 2, 2)) A_init = self.A_init A = self.A p0 = gamma[0] ntaxa = w.shape[1] tpts = w.shape[0] # log of invalid values are dealt with automatically # so let's turn off the warnings here. the assertions # should catch any unexpected errors np.seterr(divide="ignore", invalid="ignore") for t in range(1, tpts): at0 = alpha[t - 1] at1 = alpha[t] gt1 = gamma[t] p = np.concatenate((z[t] + var_Z, [0])) p = np.tile(p, ntaxa).reshape(ntaxa, ntaxa) w0 = np.tile(np.log(w[t]), ntaxa).reshape(ntaxa, ntaxa) np.fill_diagonal(w0, 0) p += w0 p = np.exp(p - logsumexp(p, axis=1, keepdims=True)) n = y[t].sum() y0 = np.tile(y[t], ntaxa).reshape(ntaxa, ntaxa) obs_1 = multinomial(y0, n, p) obs_1[:ntaxa - 1] += norm.logpdf(loc=x[t], scale=np.sqrt(var_Z), x=z[t]) np.fill_diagonal(p, 0) p /= p.sum(axis=1, keepdims=True) obs_0 = np.zeros(ntaxa) obs_0[y[t] > 0] = -np.inf obs_0[y[t] == 0] = multinomial(y0, n, p)[y[t] == 0] ids = np.logical_and(y[t] > 0, y[t - 1] > 0) if np.sum(ids) > 0: w0_w1[ids, t, 0, 1] = 0 w0_w1[ids, t, 0, 0] = 0 w0_w1[ids, t, 1, 1] = 1 w0_w1[ids, t, 1, 0] = 0 ids = np.logical_and(y[t] > 0, y[t - 1] == 0) if np.sum(ids) > 0: w0_w1[ids, t, 0, 1] = np.log(1 - at0[ids]) + obs_1[ids] + np.log( gt1[ids]) + np.log(A[ids, 0, 1]) - np.log( at1[ids]) w0_w1[ids, t, 0, 0] = -np.inf w0_w1[ids, t, 1, 1] = np.log( at0[ids]) + obs_1[ids] + np.log(gt1[ids]) + np.log( A[ids, 1, 1]) - np.log(at1[ids]) w0_w1[ids, t, 1, 0] = -np.inf log_denom = logsumexp(w0_w1[ids, t], axis=(1, 2), keepdims=True) w0_w1[ids, t] = np.exp(w0_w1[ids, t] - log_denom) assert np.all( np.abs(w0_w1[ids, t].sum(axis=(1, 2)) - 1) < 1e-2), w0_w1[ids, t] ids = np.logical_and(y[t] == 0, y[t - 1] > 0) if np.sum(ids) > 0: w0_w1[ids, t, 0, 1] = -np.inf w0_w1[ids, t, 0, 0] = -np.inf w0_w1[ids, t, 1, 1] = np.log( at0[ids]) + obs_1[ids] + np.log(gt1[ids]) + np.log( A[ids, 1, 1]) - np.log(at1[ids]) w0_w1[ids, t, 1, 0] = np.log( at0[ids]) + obs_0[ids] + np.log(1 - gt1[ids]) + np.log( A[ids, 1, 0]) - np.log(1 - at1[ids]) w0_w1[np.logical_and(ids, gt1 == 0), t, 1, 1] = -np.inf w0_w1[np.logical_and(ids, gt1 == 1), t, 1, 0] = -np.inf log_denom = logsumexp(w0_w1[ids, t], axis=(1, 2), keepdims=True) w0_w1[ids, t] = np.exp(w0_w1[ids, t] - log_denom) assert np.all( np.abs(w0_w1[ids, t].sum(axis=(1, 2)) - 1) < 1e-2 ), str(w0_w1[ids, t]) + "\n" + str(log_denom) ids = np.logical_and(y[t] == 0, y[t - 1] == 0) if np.sum(ids) > 0: w0_w1[ids, t, 0, 1] = np.log(1 - at0[ids]) + obs_1[ids] + np.log( gt1[ids]) + np.log(A[ids, 0, 1]) - np.log( at1[ids]) w0_w1[ids, t, 0, 0] = np.log(1 - at0[ids]) + obs_0[ids] + np.log( 1 - gt1[ids]) + np.log( A[ids, 0, 0]) - np.log(1 - at1[ids]) w0_w1[ids, t, 1, 1] = np.log( at0[ids]) + obs_1[ids] + np.log(gt1[ids]) + np.log( A[ids, 1, 1]) - np.log(at1[ids]) w0_w1[ids, t, 1, 0] = np.log( at0[ids]) + obs_0[ids] + np.log(1 - gt1[ids]) + np.log( A[ids, 1, 0]) - np.log(1 - at1[ids]) w0_w1[np.logical_and(ids, gt1 == 0), t, 0, 1] = -np.inf w0_w1[np.logical_and(ids, gt1 == 0), t, 1, 1] = -np.inf w0_w1[np.logical_and(ids, gt1 == 1), t, 0, 0] = -np.inf w0_w1[np.logical_and(ids, gt1 == 1), t, 1, 0] = -np.inf log_denom = logsumexp(w0_w1[ids, t], axis=(1, 2), keepdims=True) w0_w1[ids, t] = np.exp(w0_w1[ids, t] - log_denom) assert np.all( np.abs(w0_w1[ids, t].sum(axis=(1, 2)) - 1) < 1e-2 ), str(w0_w1[ids, t]) + "\n" + str(log_denom) assert np.all(w0_w1[:, t, 0, 1] >= 0) and np.all( w0_w1[:, t, 0, 1] <= 1) assert np.all(w0_w1[:, t, 1, 1] >= 0) and np.all( w0_w1[:, t, 1, 1] <= 1) # reset error messages np.seterr(divide="warn", invalid="warn") return w0_w1
def log_prior(self, x): log_p = np.log(self.in_bounds(x)) log_p += norm.logpdf(x['y']) return log_p
Date: August, 2019 [email protected] ========================================================================= """ from scipy.stats import norm import numpy as np import matplotlib.pyplot as plt #%% print(norm.pdf(0)) mean = 5 std = 10 print(norm.pdf(0, mean, std)) array = np.random.randn(1000) plt.figure(1) plt.scatter(array, norm.pdf(array)) plt.figure(2) plt.scatter(array, norm.logpdf(array)) plt.figure(3) plt.scatter(array, norm.cdf(array)) plt.figure(4) plt.scatter(array, norm.logcdf(array))
def pred_ll(y_test, mu, sigma): return norm.logpdf(y_test, loc=mu, scale=sigma).mean()