def test_simple_inference_can_be_run():
n_samples = 4
true_params = [0.5, 0.5] # t1, t2
for sampling_type in ["grid", "uniform", "BO"]:
params = BolfiParams(bounds=((0, 1), (0, 1)),
n_samples=n_samples,
n_initial_evidence=2,
sampling_type=sampling_type,
grid_tics=[[0.25, 0.75], [0.33, 0.66]],
seed=1,
simulator_node_name="MA2",
discrepancy_node_name="d")
model = get_model(n_obs=1000, true_params=true_params, seed_obs=1)
results = list()
bf = BolfiFactory(model, params)
exp = bf.get()
exp.do_sampling()
exp.compute_samples_and_ML()
assert len(exp.samples) == n_samples
assert exp.ML_val is not None
exp.compute_posterior()
assert exp.post is not None
exp.compute_MAP()
assert exp.MAP_val is not None
ML_sim = exp.simulate_data(exp.ML)
MAP_sim = exp.simulate_data(exp.MAP)
ML_disc = exp.compute_discrepancy_with_data(exp.ML, ML_sim)
MAP_disc = exp.compute_discrepancy_with_data(exp.MAP, MAP_sim)
def test_multi_parameter_linear_adjustment():
"""A regression test against values obtained in the notebook."""
seed = 20170511
threshold = 0.2
batch_size = 1000
n_samples = 500
m = ma2.get_model(true_params=[0.6, 0.2], seed_obs=seed)
summary_names = ['S1', 'S2']
parameter_names = ['t1', 't2']
linear_adjustment = LinearAdjustment()
res = elfi.Rejection(
m['d'],
batch_size=batch_size,
output_names=['S1', 'S2'],
# output_names=summary_names, # fails ?!?!?
seed=seed).sample(
n_samples, threshold=threshold)
adjusted = adjust_posterior(
model=m,
sample=res,
parameter_names=parameter_names,
summary_names=summary_names,
adjustment=linear_adjustment)
t1 = adjusted.outputs['t1']
t2 = adjusted.outputs['t2']
t1_mean, t1_var = (0.51606048286584782, 0.017253007645871756)
t2_mean, t2_var = (0.15805189695581101, 0.028004406914362647)
assert np.allclose(_statistics(t1), (t1_mean, t1_var))
assert np.allclose(_statistics(t2), (t2_mean, t2_var))
def setup_ma2_with_informative_data():
true_params = OrderedDict([('t1', .6), ('t2', .2)])
n_obs = 100
# In our implementation, seed 4 gives informative (enough) synthetic observed
# data of length 100 for quite accurate inference of the true parameters using
# posterior mean as the point estimate
m = ma2.get_model(n_obs=n_obs, true_params=true_params.values(), seed_obs=4)
return m, true_params
def test_observed():
true_params = [.6, .2]
m = ema2.get_model(100, true_params=true_params)
y = m.observed['MA2']
S1 = m.get_reference('S1')
S2 = m.get_reference('S2')
S1_observed = ema2.autocov(y)
S2_observed = ema2.autocov(y, 2)
assert np.array_equal(S1.observed, S1_observed)
assert np.array_equal(S2.observed, S2_observed)
def test_sample_object_to_dict():
data_rej = OrderedDict()
data_smc = OrderedDict()
m = get_model(n_obs=100, true_params=[.6, .2])
batch_size, n = 1, 2
schedule = [0.7, 0.2, 0.05]
rej = elfi.Rejection(m['d'], batch_size=batch_size)
res_rej = rej.sample(n, threshold=0.1)
smc = elfi.SMC(m['d'], batch_size=batch_size)
res_smc = smc.sample(n, schedule)
sample_object_to_dict(data_rej, res_rej)
sample_object_to_dict(data_smc, res_smc, skip='populations')
assert any(x not in data_rej for x in ['meta', 'output']) is True
assert any(x not in data_smc for x in ['meta', 'output', 'populations']) is True
def test_pickle_ma2():
m = ma2.get_model()
d = m.get_reference('d')
np.random.seed(0)
res1 = d.generate(10)
serialized = pickle.dumps(m)
m = pickle.loads(serialized)
d = m.get_reference('d')
np.random.seed(0)
res2 = d.generate(10)
assert np.array_equal(res1, res2)
def test_simple_inference_experiment_can_be_run():
n_samples = 4
true_params = [0.5, 0.5] # t1, t2
params = BolfiParams(bounds=((0, 1), (0, 1)),
n_samples=n_samples,
n_initial_evidence=2,
sampling_type="uniform",
seed=1,
simulator_node_name="MA2",
discrepancy_node_name="d")
model = get_model(n_obs=1000, true_params=true_params, seed_obs=1)
bf = BolfiFactory(model, params)
test_data = model.generate(1)["MA2"][0]
ground_truth = {"t1": 0.5, "t2": 0.5}
inference_experiment(bf, ground_truth=ground_truth, test_data=test_data)
def test_simple_inference_can_be_run_consistently():
for sampling_type in ["grid", "uniform", "BO"]:
params = BolfiParams(bounds=((0, 1), (0, 1)),
n_samples=4,
n_initial_evidence=2,
sampling_type=sampling_type,
grid_tics=[[0.25, 0.75], [0.33, 0.66]],
seed=1,
discrepancy_node_name="d")
model = get_model()
results = list()
bf = BolfiFactory(model, params)
for i in range(2):
post = bf.get().run()
results.append(post.ML[0])
np.testing.assert_array_almost_equal(results[0], results[1])
def test_romc3():
"""Test that ROMC provides sensible samples at the MA2 example."""
# load built-in model
seed = 1
np.random.seed(seed)
model = ma2.get_model(seed_obs=seed)
# define romc inference method
bounds = [(-2, 2), (-2, 2)]
romc = elfi.ROMC(model, bounds=bounds, discrepancy_name="d")
# solve problems
n1 = 100
seed = 21
romc.solve_problems(n1=n1, seed=seed)
# estimate regions
eps_filter = .02
romc.estimate_regions(eps_filter=eps_filter,
fit_models=True,
eps_cutoff=0.1)
# sample from posterior
n2 = 50
romc.sample(n2=n2)
romc_mean = romc.result.sample_means_array
romc_cov = romc.result.samples_cov()
# Inference with Rejection
N = 10000
rej = elfi.Rejection(model,
discrepancy_name="d",
batch_size=10000,
seed=seed)
result = rej.sample(N, threshold=.1)
rejection_mean = result.sample_means_array
rejection_cov = np.cov(result.samples_array.T)
# assert summary statistics of samples match the ground truth
assert np.allclose(romc_mean, rejection_mean, atol=.1)
assert np.allclose(romc_cov, rejection_cov, atol=.1)
import elfi from elfi.examples import ma2 # load the model from elfi.examples model = ma2.get_model() # setup and run rejection sampling rej = elfi.Rejection(model['d'], batch_size=10000) result = rej.sample(1000, quantile=0.01) # show summary of results on stdout result.summary()
def test_implementing_new_algorithm():
import numpy as np
from elfi.methods.parameter_inference import ParameterInference
from elfi.methods.results import Sample
import elfi.examples.ma2 as ma2
class CustomMethod(ParameterInference):
def __init__(self, model, discrepancy_name, threshold, **kwargs):
# Create a name list of nodes whose outputs we wish to receive
output_names = [discrepancy_name] + model.parameter_names
super(CustomMethod, self).__init__(model, output_names, **kwargs)
self.threshold = threshold
self.discrepancy_name = discrepancy_name
# Prepare lists to push the filtered outputs into
self.state['filtered_outputs'] = {
name: []
for name in output_names
}
def set_objective(self, n_sim):
self.objective['n_sim'] = n_sim
def update(self, batch, batch_index):
super(CustomMethod, self).update(batch, batch_index)
# Make a filter mask (logical numpy array) from the distance array
filter_mask = batch[self.discrepancy_name] <= self.threshold
# Append the filtered parameters to their lists
for name in self.output_names:
values = batch[name]
self.state['filtered_outputs'][name].append(
values[filter_mask])
def extract_result(self):
filtered_outputs = self.state['filtered_outputs']
outputs = {
name: np.concatenate(filtered_outputs[name])
for name in self.output_names
}
return Sample(method_name='CustomMethod',
outputs=outputs,
parameter_names=self.parameter_names,
discrepancy_name=self.discrepancy_name,
n_sim=self.state['n_sim'],
threshold=self.threshold)
# Below is from the part where we demonstrate iterative advancing
# Run it
m = ma2.get_model()
custom_method = CustomMethod(m, 'd', threshold=.1, batch_size=1000)
# Continue inference from the previous state (with n_sim=2000)
custom_method.infer(n_sim=4000)
# Or use it iteratively
custom_method.set_objective(n_sim=6000)
custom_method.iterate()
assert custom_method.finished == False
# Investigate the current state
custom_method.extract_result()
custom_method.iterate()
assert custom_method.finished
custom_method.extract_result()
(mean + std - range[0]) / (range[1] - range[0]),
color="k",
linestyle="--",
label=r"$\sigma = %.3f$" % (std))
plt.xlabel(xlabel)
plt.ylabel(ylabel)
plt.ylim(ylim)
plt.legend()
plt.savefig(savepath, bbox_inches='tight')
plt.show(block=False)
# Set seed for reproducibility
seed = 1
np.random.seed(seed)
model = ma2.get_model(seed_obs=seed)
# plot prior samples
x = model.generate(1000)
plt.figure()
plt.title("Samples from the prior")
plt.xlabel(r"$\theta_1$")
plt.ylabel(r"$\theta_2$")
plt.plot(x["t1"], x["t2"], "bo")
plt.savefig(os.path.join(prepath, "mae2_prior_samples.png"),
bbox_inches="tight")
plt.show(block=False)
####### ROMC with gradients ################
n1 = 1000
