def compute_mse(self, ep_x_data):
self.x_pred = {x_id: data["r"] for x_id, data in ep_x_data.items()}
# MSE estimated by ep
self.mse_ep = {x_id: data["v"] for x_id, data in ep_x_data.items()}
# Real MSE
self.mse = min(mean_squared_error(self.x_true['x'], self.x_pred['x']),
mean_squared_error(self.x_true['x'], -self.x_pred['x']))
print(f"mse_ep: {self.mse_ep['x']:.3f} mse: {self.mse: .3f}")
return self.mse_ep['x'], self.mse
def run_benchmark(alpha, algo, seed):
# create scenario
N, rho, noise_var = 1000, 0.05, 1e-2
M = int(alpha * N)
A = GaussianEnsemble(M=M, N=N).generate()
t0 = time()
model = (GaussBernoulliPrior(size=N, rho=rho) @ V("x") @ LinearChannel(A)
@ V("z") @ GaussianChannel(var=noise_var) @ O("y")).to_model()
t1 = time()
record = {"svd_time": t1 - t0} # svd precomputation time
scenario = BayesOptimalScenario(model, x_ids=["x"])
scenario.setup(seed)
y = scenario.observations["y"]
# run algo
t0 = time()
if algo == "SE":
x_data = scenario.run_se(max_iter=1000, damping=0.1)
record["mse"] = x_data["x"]["v"]
record["n_iter"] = x_data["n_iter"]
if algo == "EP":
x_data = scenario.run_ep(max_iter=1000, damping=0.1)
x_pred = x_data["x"]["r"]
record["n_iter"] = x_data["n_iter"]
if algo == "LassoCV":
lasso = LassoCV(cv=5)
lasso.fit(A, y)
x_pred = lasso.coef_
record["param_scikit"] = lasso.alpha_
record["n_iter"] = lasso.n_iter_
if algo == "Lasso":
optim = pd.read_csv("optimal_param_lasso.csv")
param_scaled = np.interp(alpha, optim["alpha"], optim["param_scaled"])
param_scikit = noise_var * param_scaled / (M * rho)
lasso = Lasso(alpha=param_scikit)
lasso.fit(A, y)
x_pred = lasso.coef_
record["param_scikit"] = param_scikit
record["n_iter"] = lasso.n_iter_
if algo == "pymc3":
with pm.Model():
ber = pm.Bernoulli("ber", p=rho, shape=N)
nor = pm.Normal("nor", mu=0, sd=1, shape=N)
x = pm.Deterministic("x", ber * nor)
likelihood = pm.Normal("y",
mu=pm.math.dot(A, x),
sigma=np.sqrt(noise_var),
observed=y)
trace = pm.sample(draws=1000, chains=1, return_inferencedata=False)
x_pred = trace.get_values('x').mean(axis=0)
t1 = time()
record["time"] = t1 - t0
if algo != "SE":
record["mse"] = mean_squared_error(x_pred, scenario.x_true["x"])
return record
def run_EP(alpha, rho, seed):
model = glm_generative(N=2000,
alpha=alpha,
ensemble_type="gaussian",
prior_type="gauss_bernoulli",
output_type="gaussian",
prior_rho=rho,
output_var=1e-10)
scenario = BayesOptimalScenario(model, x_ids=["x"])
scenario.setup(seed)
x_data = scenario.run_ep(max_iter=200)
x_pred = x_data["x"]["r"]
mse = mean_squared_error(x_pred, scenario.x_true["x"])
return dict(source="EP", v=mse)
def mse_lasso(alpha, param_scaled, seed):
# create scenario
N, rho, noise_var = 1000, 0.05, 1e-2
M = int(alpha * N)
A = GaussianEnsemble(M=M, N=N).generate()
model = (GaussBernoulliPrior(size=N, rho=rho) @ V("x") @ LinearChannel(A)
@ V("z") @ GaussianChannel(var=noise_var) @ O("y")).to_model()
scenario = BayesOptimalScenario(model, x_ids=["x"])
scenario.setup(seed)
y = scenario.observations["y"]
# run lasso
param_scikit = noise_var * param_scaled / (M * rho)
lasso = Lasso(alpha=param_scikit)
lasso.fit(A, y)
x_pred = lasso.coef_
mse = mean_squared_error(x_pred, scenario.x_true["x"])
return mse
def plot_truth_vs_prediction(self, x_pred, save_fig=False, block=False):
assert self.N == 784
v_star = self.x_true['x'].reshape(28, 28)
v_hat = x_pred.reshape(28, 28)
fig, axes = plt.subplots(1, 3, figsize=(8, 8))
axes[0].imshow(v_star, cmap='Greys')
if self.model_params['name'] == 'inpainting':
y_true = self.y_inp.reshape(28, 28)
elif self.model_params['name'] in ['denoising']:
y_true = self.y_true['y'].reshape(28, 28)
else:
y_true = v_star
axes[1].imshow(y_true, cmap='Greys')
axes[2].imshow(v_hat, cmap='Greys')
axes[0].set_xlabel(r'$x^\star$', Fontsize=25)
axes[1].set_xlabel(r'$x_{\rm obs}$', Fontsize=25)
axes[2].set_xlabel(r'$\hat{x}$', Fontsize=25)
mse = mean_squared_error(self.x_true['x'], x_pred)
plt.title(f'MSE:{mse:.3f}')
plt.tight_layout()
axes[0].set_xticks([]), axes[0].set_yticks([])
axes[1].set_xticks([]), axes[1].set_yticks([])
axes[2].set_xticks([]), axes[2].set_yticks([])
# Save
id = int(time.time())
file_name = f"./Images/{self.model_params['name']}/{self.data_params['name']}_{self.prior_params['name']}_{self.prior_params['id']}_Delta{self.Delta:.3f}_alpha{self.model_params['alpha']:.3f}_{id}.pdf"
if save_fig:
plt.savefig(file_name,
format='pdf',
dpi=1000,
bbox_inches="tight",
pad_inches=0.1)
# Show
if self.plot_truth_vs_pred:
if block:
plt.show(block=False)
input('Press enter to continue')
else:
plt.show()
plt.close()
# %%
# Create a sparse teacher
N = 250
alpha = 1
rho = 0.1
Delta = 1e-2
teacher = SparseTeacher(N=N, alpha=alpha, rho=rho, Delta=Delta)
sample = (teacher.model).sample()
# %%
prior = GaussBernouilliPrior(size=(N, ), rho=rho)
student = prior @ V(id="x") @ LinearChannel(W=teacher.A) @ V(
id='z') @ GaussianLikelihood(y=sample['y'], var=Delta)
student = student.to_model_dag()
student = student.to_model()
student.plot()
# %%
max_iter = 20
damping = 0.1
ep = ExpectationPropagation(student)
ep.iterate(max_iter=max_iter,
damping=damping,
callback=EarlyStopping(tol=1e-8))
data_ep = ep.get_variables_data(['x'])
mse = mean_squared_error(data_ep['x']['r'], sample['x'])
print(mse)
