def predict(self, X, out: str = 'latent'):
"""
Predict out:
'latent' : E[y* | X]
'censored' : E[y | X]
'truncated' : E[y | y>0,X]
'P(censored)' : P(y=0 | X)
'P(uncensored)' : P(y>0 | X)
"""
beta, sigma = self.theta[0:len(self.theta) - 1], self.theta[-1]
normal = distributions.Normal(mu=0, sigma=1)
XB = self.X @ beta
cdf_eval, pdf_eval = normal.cdf(XB / sigma), normal.pdf(XB / sigma)
if out == 'latent':
return XB
elif out == 'censored':
return XB * cdf_eval + sigma * pdf_eval
elif out == 'truncated':
return XB + sigma * (pdf_eval / cdf_eval)
elif out == 'P(censored)':
return 1 - cdf_eval
elif out == 'P(uncensored)':
return cdf_eval
else:
raise ValueError(
'"out" argument must be one of ["latent", "censored", "truncated", "P(censored)", "P(uncensored)"]'
)
def __init__(self, init, data, init_proposal_sd, prior=None):
self.state = init
self.data = data
if prior is None:
self.prior = dist.Normal(0, 10)
self.tuners = StateTuner(init_proposal_sd)
def set_target_dist(dist_type, thetas, rewards):
if dist_type == 'Normal':
return distributions.Normal(thetas)
elif dist_type == 'Bandit':
return distributions.BanditDistribution(thetas, rewards)
else:
print('Unsupported Arguments Types.')
exit()
def sample_std_normal(n, seed=1):
np.random.seed(seed)
tuner = mcmc.Tuner(1.0)
xs = np.zeros(n)
x = 0.0
for b in range(n):
x = mcmc.ametropolis(x, dist.Normal(0, 1).lpdf, tuner)
# x = mcmc.ametropolis(x, lpdf_std_normal, tuner) # faster
xs[b] = x
return xs
def objective_function(self, theta: np.ndarray):
"""
negative log likelihood
theta: {beta, sigma}
"""
beta, sigma = theta[0:len(theta) - 1], theta[-1]
normal = distributions.Normal(mu=0, sigma=1)
XB = self.X @ beta
f_0 = (self.y == 0) * np.log(1 - normal.cdf(XB / sigma))
f_1 = (self.y > 0) * (
(-1 / 2) * np.log(2 * np.pi) - np.log(sigma**2) / 2 -
(self.y - XB)**2 / (2 * sigma**2))
return (-1) * np.sum(
f_0 + f_1, axis=0) / self.X.shape[0] # sum log likelihood (meaned)
def compute_marginal_effects(self,
X: np.ndarray,
theta: np.ndarray,
out: str = 'latent'):
"""
Calculate marginal effects given X and theta
out:
'latent' : dE[y*|X]/dx
'censored' : dE[y|X]/dx
"""
if out == 'latent':
return theta[0:len(theta) - 1] # beta
elif out == 'censored':
beta, sigma = theta[0:len(theta) - 1], theta[-1]
normal = distributions.Normal(mu=0, sigma=1)
return normal.cdf((X @ beta) / sigma) * beta
else:
raise ValueError(
'Input argument "out" must be either "latent" or "censored"')
def hessian(self, theta: np.ndarray):
"""
Compute and construct hessian at theta,
the formulae found in Calzolari & Fiorentini 1993
"""
beta, sigma = theta[0:len(theta) - 1], theta[-1]
normal = distributions.Normal(mu=0, sigma=1)
XB = self.X @ beta
r = self.y - XB
cdf_eval, pdf_eval = normal.cdf(XB / sigma), normal.pdf(XB / sigma)
# Hessian blocks
dL_dBdB = np.zeros(shape=(self.X.shape[1],
self.X.shape[1])) # (K x K), d^2L/dBdB'
dL_dsdB = np.zeros(shape=(self.X.shape[1])) # (K x 1), d^2L/ds^2dB'
dL_dsds = np.zeros(shape=(1)) # (1 x 1), d^2L/d^2s^2
for i in range(self.X.shape[0]):
xixi = np.outer(self.X[i, :], self.X[i, :]) # (K x K)
dL_dBdB += (-1) * (self.y[i] > 0) * (1 / sigma**2) * xixi - (
self.y[i]
== 0) * (1 / sigma) * (pdf_eval[i] / (1 - cdf_eval[i])**2) * (
(pdf_eval[i] / sigma) - (1 / sigma**2) *
(1 - cdf_eval[i]) * XB[i]) * xixi # (K x K)
dL_dsdB += (-1) * (self.y[i] > 0) * (
1 / sigma**2) * r[i] * self.X[i, :] - (self.y[i] == 0) * (
1 /
(2 * sigma**3)) * (pdf_eval[i] / (1 - cdf_eval[i])**2) * (
(1 / sigma**2) * (1 - cdf_eval[i]) * XB[i]**2 -
(1 - cdf_eval[i]) - ((XB[i] * pdf_eval[i]) /
sigma)) * self.X[i, :] # (K x 1)
dL_dsds += (-1) * (self.y[i] > 0) * (1 / sigma**6) * r[i]**2 - (
self.y[i] == 0) * (1 / (4 * sigma**5)) * (
pdf_eval[i] / (1 - cdf_eval[i])**2) * (
(1 / sigma**2) * (1 - cdf_eval[i]) * (XB[i]**3) - 3 *
(1 - cdf_eval[i]) * (XB[i]) -
(XB[i] * pdf_eval[i] / sigma)) # (1 x 1)
return np.concatenate((np.concatenate(
(dL_dBdB, np.reshape(dL_dsdB, newshape=(-1, 1))), axis=1),
np.reshape(np.concatenate(
(dL_dsdB, dL_dsds), axis=0),
newshape=(1, -1))),
axis=0)
def score(self, theta: np.ndarray):
"""
Implement objective function derivative
"""
beta, sigma = theta[0:len(theta) - 1], theta[-1]
normal = distributions.Normal(mu=0, sigma=1)
XB = self.X @ beta
r = self.y - XB
cdf_eval, pdf_eval = normal.cdf(XB / sigma), normal.pdf(XB / sigma)
partial_beta = (1 / sigma**2) * np.sum(np.reshape(
((self.y > 0) * r - (self.y == 0) * ((sigma * pdf_eval) /
(1 - cdf_eval))),
newshape=(-1, 1)) * self.X,
axis=0) # (K x 1)
partial_sigma = np.sum(
(self.y > 0) * ((-1) / (2 * sigma**2) + r**2 /
(2 * sigma**4)) + (self.y == 0) * XB /
(2 * sigma**3) * pdf_eval / (1 - cdf_eval)) # (1 x 1)
#print('score: ', np.concatenate((partial_beta, np.atleast_1d(partial_sigma)), axis=0))
return -np.concatenate((partial_beta, np.atleast_1d(partial_sigma)),
axis=0) # the full score vector ((K+1) x 1)
def outer_product(self, theta: np.ndarray):
"""
Compute outer product of gradients at theta
the formulae found in Calzolari & Fiorentini 1993
"""
beta, sigma = theta[0:len(theta) - 1], theta[-1]
normal = distributions.Normal(mu=0, sigma=1)
XB = self.X @ beta
r = self.y - XB
cdf_eval, pdf_eval = normal.cdf(XB / sigma), normal.pdf(XB / sigma)
# Outer Product blocks
dL_dBdB = np.zeros(shape=(self.X.shape[1],
self.X.shape[1])) # (K x K), d^2L/dBdB'
dL_dsdB = np.zeros(shape=(self.X.shape[1])) # (K x 1), d^2L/ds^2dB'
dL_dsds = np.zeros(shape=(1)) # (1 x 1), d^2L/d^2s
for i in range(self.X.shape[0]):
xixi = np.outer(self.X[i, :], self.X[i, :]) # (K x K)
dL_dBdB += (-1) * (1 / sigma**2) * (pdf_eval[i] * (XB[i] / sigma) -
(pdf_eval[i]**2 /
(1 - cdf_eval[i])) -
cdf_eval[i]) * xixi
dL_dsdB += (-1) * (1 / (2 * sigma**3)) * (
pdf_eval[i] * (XB[i] / sigma)**2 + pdf_eval[i] -
(pdf_eval[i]**2 /
(1 - cdf_eval[i])) * (XB[i] / sigma)) * self.X[i, :]
dL_dsds += (-1) * (1 / (4 * sigma**4)) * (
pdf_eval[i] * (XB[i] / sigma)**3 + pdf_eval[i] *
(XB[i] / sigma) - (pdf_eval[i]**2 / (1 - cdf_eval[i])) *
(XB[i] / sigma) - 2 * cdf_eval[i])
return np.concatenate((np.concatenate(
(dL_dBdB, np.reshape(dL_dsdB, newshape=(-1, 1))), axis=1),
np.reshape(np.concatenate(
(dL_dsdB, dL_dsds), axis=0),
newshape=(1, -1))),
axis=0)
import numba
import numpy as np
import mcmc
import distributions
from tqdm import trange
log_prob = lambda x: distributions.Normal(0, 1).lpdf(x)
def f(x, n=1e6):
for i in trange(int(n)):
x = mcmc.metropolis(x, log_prob, 1.0)
print(x)
if __name__ == '__main__':
f(1.0, 1)
f(1.0)
def f(x, n):
log_prob = distributions.Normal(0, 1).lpdf
for _ in range(n):
x = mcmc.metropolis(x, log_prob, 1.0)
return x
def b2_prior():
return dist.Normal(0, 10)