def setup_module():
"""Run an analysis and create a database.
Called once at the beginning.
"""
def model(p):
return {
'ss0': p['p0'] + 0.1 * np.random.uniform(),
'ss1': p['p1'] + 0.1 * np.random.uniform(),
}
p_true = {'p0': 3, 'p1': 4}
observation = {'ss0': p_true['p0'], 'ss1': p_true['p1']}
limits = {'p0': (0, 5), 'p1': (1, 8)}
prior = pyabc.Distribution(
**{
key: pyabc.RV('uniform', limits[key][0], limits[key][1] -
limits[key][0])
for key in p_true.keys()
})
distance = pyabc.PNormDistance(p=2)
abc = pyabc.ABCSMC(model, prior, distance, population_size=50)
abc.new(db_path, observation)
abc.run(minimum_epsilon=0.1, max_nr_populations=4)
def test_pdf_norm_methods_integration():
"""Test integration of pdf normalization methods in ABCSMC."""
def model(par):
return {'s0': par['p0'] + np.array([0.3, 0.7])}
x_0 = {'s0': np.array([0.4, -0.6])}
for pdf_norm in [
pyabc.pdf_norm_max_found,
pyabc.pdf_norm_from_kernel,
pyabc.ScaledPDFNorm(),
]:
# just run
acceptor = pyabc.StochasticAcceptor(pdf_norm_method=pdf_norm)
eps = pyabc.Temperature()
distance = pyabc.IndependentNormalKernel(var=np.array([1, 1]))
prior = pyabc.Distribution(p0=pyabc.RV('uniform', -1, 2))
abc = pyabc.ABCSMC(model,
prior,
distance,
eps=eps,
acceptor=acceptor,
population_size=20)
abc.new(pyabc.create_sqlite_db_id(), x_0)
abc.run(max_nr_populations=3)
def prior(n_percent):
with open("settings_siwik.config") as config_file:
config = json.load(config_file)
def paramRV(range):
return pyabc.RV("uniform", range[0], range[1])
def dist(val, n):
inter_half = 0.01 * n * val
return val - inter_half / 2, inter_half
dist_n_percent = functools.partial(dist, n=n_percent)
return pyabc.Distribution(
lambda_p=paramRV(dist_n_percent(
config["equation_params"]["lambda_p"])),
K=paramRV(dist_n_percent(config["equation_params"]["K"])),
k_qpp=paramRV(dist_n_percent(config["equation_params"]["k_qpp"])),
k_pq=paramRV(dist_n_percent(config["equation_params"]["k_pq"])),
y_p=paramRV(dist_n_percent(config["equation_params"]["y_p"])),
y_q=paramRV(dist_n_percent(config["equation_params"]["y_q"])),
delta_qp=paramRV(dist_n_percent(
config["equation_params"]["delta_qp"])),
KDE=paramRV(dist_n_percent(config["equation_params"]["KDE"])),
Q0=paramRV(dist_n_percent(config["initial_state"]["Q0"])),
P0=paramRV(dist_n_percent(config["initial_state"]["P0"])),
)
def test_redis_catch_error():
def model(pars):
if np.random.uniform() < 0.1:
raise ValueError("error")
return {'s0': pars['p0'] + 0.2 * np.random.uniform()}
def distance(s0, s1):
return abs(s0['s0'] - s1['s0'])
prior = pyabc.Distribution(p0=pyabc.RV("uniform", 0, 10))
sampler = RedisEvalParallelSamplerServerStarter(batch_size=3,
workers=1,
processes_per_worker=1)
try:
abc = pyabc.ABCSMC(model,
prior,
distance,
sampler=sampler,
population_size=10)
db_file = "sqlite:///" + os.path.join(tempfile.gettempdir(), "test.db")
data = {'s0': 2.8}
abc.new(db_file, data)
abc.run(minimum_epsilon=.1, max_nr_populations=3)
finally:
sampler.shutdown()
def test_rpy2(sampler):
# run the notebook example
r_file = "doc/examples/myRModel.R"
r = pyabc.external.R(r_file)
r.display_source_ipython()
model = r.model("myModel")
distance = r.distance("myDistance")
sum_stat = r.summary_statistics("mySummaryStatistics")
data = r.observation("mySumStatData")
prior = pyabc.Distribution(meanX=pyabc.RV("uniform", 0, 10),
meanY=pyabc.RV("uniform", 0, 10))
abc = pyabc.ABCSMC(model,
prior,
distance,
summary_statistics=sum_stat,
sampler=sampler,
population_size=5)
db = pyabc.create_sqlite_db_id(file_="test_external.db")
abc.new(db, data)
history = abc.run(minimum_epsilon=0.9, max_nr_populations=2)
history.get_weighted_sum_stats_for_model(m=0, t=1)[1][0]["cars"].head()
# try load
id_ = history.id
abc = pyabc.ABCSMC(model,
prior,
distance,
summary_statistics=sum_stat,
sampler=sampler,
population_size=6)
# shan't even need to pass the observed data again
abc.load(db, id_)
abc.run(minimum_epsilon=0.1, max_nr_populations=2)
def test_early_stopping():
"""Basic test whether an early stopping pipeline works.
Heavily inspired by the `early_stopping` notebook.
"""
prior = pyabc.Distribution(step_size=pyabc.RV("uniform", 0, 10))
n_steps = 30
gt_step_size = 5
gt_trajectory = simulate(n_steps, gt_step_size)
model = MyStochasticProcess(n_steps=n_steps,
gt_step_size=gt_step_size,
gt_trajectory=gt_trajectory)
abc = pyabc.ABCSMC(
models=model,
parameter_priors=prior,
distance_function=pyabc.NoDistance(),
population_size=30,
transitions=pyabc.LocalTransition(k_fraction=0.2),
eps=pyabc.MedianEpsilon(300, median_multiplier=0.7),
)
# initializing eps manually is necessary as we only have an integrated
# model
# TODO automatically iniitalizing would be possible, e.g. using eps = inf
abc.new(pyabc.create_sqlite_db_id())
abc.run(max_nr_populations=3)
def test_export():
"""Test database export.
Just calls export and does some very basic checks.
"""
# simple problem
def model(p):
return {"y": p["p"] + 0.1 * np.random.normal()}
prior = pyabc.Distribution(p=pyabc.RV("uniform", -1, 2))
distance = pyabc.PNormDistance()
try:
# run
db_file = tempfile.mkstemp(suffix=".db")[1]
abc = pyabc.ABCSMC(model, prior, distance, population_size=100)
abc.new("sqlite:///" + db_file, model({"p": 0}))
abc.run(max_nr_populations=3)
# export history
for fmt in ["csv", "json", "html", "stata"]:
out_file = tempfile.mkstemp()[1]
try:
pyabc.storage.export(db_file, out=out_file, out_format=fmt)
assert os.path.exists(out_file)
assert os.stat(out_file).st_size > 0
finally:
if os.path.exists(out_file):
os.remove(out_file)
finally:
if os.path.exists(db_file):
os.remove(db_file)
def test_simple_function_acceptor():
def distance(x, x_0):
return sum(abs(x[key] - x_0[key]) for key in x_0)
def dummy_accept(dist, eps, x, x_0, t, par):
d = dist(x, x_0)
return AcceptorResult(d, d < eps(t))
x = {'s0': 1, 's1': 0}
y = {'s0': 2, 's1': 2}
acceptor = pyabc.SimpleFunctionAcceptor(dummy_accept)
ret = acceptor(distance_function=distance,
eps=lambda t: 0.1,
x=x,
x_0=y,
t=0,
par=None)
assert isinstance(ret, AcceptorResult)
assert ret.distance == 3
# test integration
def model(par):
return {'s0': par['p0'] + 1, 's1': 42}
prior = pyabc.Distribution(p0=pyabc.RV('uniform', -5, 10))
abc = pyabc.ABCSMC(model, prior, distance, population_size=2)
abc.new(pyabc.create_sqlite_db_id(), model({'p0': 1}))
h = abc.run(max_nr_populations=2)
df = h.get_weighted_distances()
assert np.isfinite(df['distance']).all()
def test_stochastic_acceptor():
acceptor = pyabc.StochasticAcceptor(
pdf_norm_method=pyabc.pdf_norm_max_found)
eps = pyabc.Temperature(initial_temperature=1)
distance = pyabc.IndependentNormalKernel(var=np.array([1, 1]))
def model(par):
return {'s0': par['p0'] + np.array([0.3, 0.7])}
x_0 = {'s0': np.array([0.4, -0.6])}
# just run
prior = pyabc.Distribution(p0=pyabc.RV('uniform', -1, 2))
abc = pyabc.ABCSMC(model,
prior,
distance,
eps=eps,
acceptor=acceptor,
population_size=10)
abc.new(pyabc.create_sqlite_db_id(), x_0)
abc.run(max_nr_populations=1, minimum_epsilon=1.)
# use no initial temperature and adaptive c
acceptor = pyabc.StochasticAcceptor()
eps = pyabc.Temperature()
abc = pyabc.ABCSMC(model,
prior,
distance,
eps=eps,
acceptor=acceptor,
population_size=10)
abc.new(pyabc.create_sqlite_db_id(), x_0)
abc.run(max_nr_populations=3, minimum_epsilon=1.)
def test_default_eps():
def model(par):
return {'s0': par['p0'] + np.random.random(), 's1': np.random.random()}
x_0 = {'s0': 0.4, 's1': 0.6}
prior = pyabc.Distribution(p0=pyabc.RV('uniform', -1, 2))
# usual setting
abc = pyabc.ABCSMC(model, prior, population_size=10)
abc.new(pyabc.create_sqlite_db_id(), x_0)
abc.run(max_nr_populations=3)
assert abc.minimum_epsilon == 0.0
# noisy setting
acceptor = pyabc.StochasticAcceptor()
eps = pyabc.Temperature()
distance = pyabc.IndependentNormalKernel(var=np.array([1, 1]))
abc = pyabc.ABCSMC(model,
prior,
distance,
eps=eps,
acceptor=acceptor,
population_size=10)
abc.new(pyabc.create_sqlite_db_id(), x_0)
abc.run(max_nr_populations=3)
assert abc.minimum_epsilon == 1.0
def run_methods(mutation_rate, fitness, epsilon=0.005, max_episodes=10, w_prior=(0,2), mu_prior=(-7,5),
smc_population_size=1000, data=(0,), methods='All', model_ss=10 ** 5, data_ss=10 ** 5,
pop_size=10**8, gen_num=10, position=None, plot=True):
"""
This is the main tool which runs multiple methods and graphs their posteriors.
"""
if methods == 'All':
methods = ['megapost', 'megadist']
if not (isinstance(data, pd.DataFrame) or isinstance(data, pd.Series)):
print("Creating dataset...")
data = simulate_data(generations_number=gen_num, wt_freqs=data, mutation_rate=mutation_rate,
population_size=pop_size, fitness=fitness, sequence_sample_size=data_ss,
plot=False)
prior_dist = pyabc.Distribution(w=pyabc.RV("uniform", w_prior[0], w_prior[1]),
mu=pyabc.RV("uniform", mu_prior[0], mu_prior[1]))
posts = dict()
if position:
print(f"position: {position}") # just for the logging
if 'megapost' in methods:
print("Inferring with mega posterior")
posts['megapost'] = infer_megapost(prior_dist=prior_dist, data=data, epsilon=epsilon,
max_episodes=max_episodes, smc_population_size=smc_population_size,
sequence_sample_size=model_ss, pop_size=pop_size)
if 'megadist' in methods:
print("Inferring with mega distance function")
posts['megadist'] = run_smc(priors=prior_dist, data=data, epsilon=len(data) * epsilon, max_episodes=max_episodes,
smc_population_size=smc_population_size, pop_size=pop_size,
sequence_sample_size=model_ss)
if plot:
plot_kdes(fitness, mutation_rate, posts)
return posts
def smc_weighted_dist(mutation_rate, fitness, max_episodes=10, w_prior=(0, 2), mu_prior=(-7, 5),
smc_population_size=1000, data=(0,), model_ss=10**4, data_ss=10**4,
pop_size=10**8, gen_num=10, position=None, plot=True):
"""
This is the main tool which runs multiple methods and graphs their posteriors.
"""
if not (isinstance(data, pd.DataFrame) or isinstance(data, pd.Series)):
print("Creating dataset...")
data = simulate_data(generations_number=gen_num, wt_freqs=data, mutation_rate=mutation_rate,
population_size=pop_size, fitness=fitness, sequence_sample_size=data_ss,
plot=False)
data = data[(data.T != 0).any()] # drop rows where all columns are zero
# TODO: does this not bias the data against lower mu & w????
prior_dist = pyabc.Distribution(w=pyabc.RV("uniform", w_prior[0], w_prior[1]),
mu=pyabc.RV("uniform", mu_prior[0], mu_prior[1]))
if position:
print(f"position: {position}") # just for the logging
df = run_smc(priors=prior_dist, data=data, epsilon=0, max_episodes=max_episodes,
smc_population_size=smc_population_size, pop_size=pop_size,
sequence_sample_size=model_ss, distance_function=weighted_l1)
if plot:
fig, ax = plt.subplots(1, 1)
plot_2d_kde_from_df(df, fitness, mutation_rate, ax, 'Weighted Distance Inference')
return df
def setup_module():
"""Set up module. Called before all tests here."""
# create and run some model
observation = {'ss0': p_true['p0'], 'ss1': p_true['p1']}
prior = pyabc.Distribution(
**{
key: pyabc.RV('uniform', limits[key][0], limits[key][1] -
limits[key][0])
for key in p_true.keys()
})
distance = pyabc.PNormDistance(p=2)
n_history = 2
sampler = pyabc.sampler.MulticoreEvalParallelSampler(n_procs=2)
for _ in range(n_history):
abc = pyabc.ABCSMC(model,
prior,
distance,
population_size=100,
sampler=sampler)
abc.new(db_path, observation)
abc.run(minimum_epsilon=.1, max_nr_populations=3)
for j in range(n_history):
history = pyabc.History(db_path)
history.id = j + 1
histories.append(history)
labels.append("Some run " + str(j))
def basic_testcase():
"""A simple test model."""
def model(p):
return {"y": p['p0'] + 0.1 * np.random.randn(10)}
prior = pyabc.Distribution(p0=pyabc.RV('uniform', -5, 10),
p1=pyabc.RV('uniform', -2, 2))
def distance(y1, y2):
return np.abs(y1['y'] - y2['y']).sum()
obs = {'y': 1}
return model, prior, distance, obs
def test_pipeline(db_file):
"""Test whole pipeline using a learned summary statistic."""
rng = np.random.Generator(np.random.PCG64(0))
def model(p):
return {"s0": p["p0"] + 1e-2 * rng.normal(size=2), "s1": rng.normal()}
prior = pyabc.Distribution(p0=pyabc.RV("uniform", 0, 1))
distance = pyabc.AdaptivePNormDistance(sumstat=PredictorSumstat(
LinearPredictor(), fit_ixs={1, 3}), )
data = {"s0": np.array([0.1, 0.105]), "s1": 0.5}
# run a little analysis
abc = pyabc.ABCSMC(model, prior, distance, population_size=100)
abc.new("sqlite:///" + db_file, data)
h = abc.run(max_total_nr_simulations=1000)
# first iteration
df0, w0 = h.get_distribution(t=0)
off0 = abs(pyabc.weighted_mean(df0.p0, w0) - 0.1)
# last iteration
df, w = h.get_distribution()
off = abs(pyabc.weighted_mean(df.p0, w) - 0.1)
assert off0 > off
# alternative run with simple distance
distance = pyabc.PNormDistance()
abc = pyabc.ABCSMC(model, prior, distance, population_size=100)
abc.new("sqlite:///" + db_file, data)
h = abc.run(max_total_nr_simulations=1000)
df_comp, w_comp = h.get_distribution()
off_comp = abs(pyabc.weighted_mean(df_comp.p0, w_comp) - 0.1)
assert off_comp > off
# alternative run with info weighting
distance = pyabc.InfoWeightedPNormDistance(
predictor=LinearPredictor(),
fit_info_ixs={1, 3},
)
abc = pyabc.ABCSMC(model, prior, distance, population_size=100)
abc.new("sqlite:///" + db_file, data)
h = abc.run(max_total_nr_simulations=1000)
df_info, w_info = h.get_distribution()
off_info = abs(pyabc.weighted_mean(df_info.p0, w_info) - 0.1)
assert off_comp > off_info
def test_sensitivity_sankey():
"""Test pyabc.visualization.plot_sensitivity_sankey`"""
sigmas = {"p1": 0.1}
def model(p):
return {
"y1": p["p1"] + 1 + sigmas["p1"] * np.random.normal(),
"y2": 2 + 0.1 * np.random.normal(size=3),
}
gt_par = {"p1": 3}
data = {"y1": gt_par["p1"] + 1, "y2": 2 * np.ones(shape=3)}
prior_bounds = {"p1": (0, 10)}
prior = pyabc.Distribution(
**{
key: pyabc.RV("uniform", lb, ub - lb)
for key, (lb, ub) in prior_bounds.items()
}, )
total_sims = 1000
# tmp files
db_file = tempfile.mkstemp(suffix=".db")[1]
scale_log_file = tempfile.mkstemp(suffix=".json")[1]
info_log_file = tempfile.mkstemp(suffix=".json")[1]
info_sample_log_file = tempfile.mkstemp()[1]
distance = pyabc.InfoWeightedPNormDistance(
p=1,
scale_function=pyabc.distance.mad,
predictor=pyabc.predictor.LinearPredictor(),
fit_info_ixs=pyabc.util.EventIxs(sims=int(0.4 * total_sims)),
scale_log_file=scale_log_file,
info_log_file=info_log_file,
info_sample_log_file=info_sample_log_file,
)
abc = pyabc.ABCSMC(model, prior, distance, population_size=100)
h = abc.new(db="sqlite:///" + db_file, observed_sum_stat=data)
abc.run(max_total_nr_simulations=total_sims)
pyabc.visualization.plot_sensitivity_sankey(
info_sample_log_file=info_sample_log_file,
t=info_log_file,
h=h,
predictor=pyabc.predictor.LinearPredictor(),
)
def test_basic(sampler: pyabc.sampler.Sampler):
"""Some basic tests."""
def model(par):
return {'s0': par['p0'] + np.random.randn(4)}
def distance(x, y):
return np.sum(x['s0'] - y['s0'])
x0 = model({'p0': 2})
prior = pyabc.Distribution(p0=pyabc.RV("uniform", 0, 10))
abc = pyabc.ABCSMC(
model, prior, distance, sampler=sampler, population_size=50
)
abc.new(pyabc.create_sqlite_db_id(), x0)
abc.run(max_nr_populations=4)
def test_reference_parameter():
def model(parameter):
return {"data": parameter["mean"] + 0.5 * np.random.randn()}
prior = pyabc.Distribution(p0=pyabc.RV("uniform", 0, 5),
p1=pyabc.RV("uniform", 0, 1))
def distance(x, y):
return abs(x["data"] - y["data"])
abc = pyabc.ABCSMC(model, prior, distance, population_size=2)
db_path = ("sqlite:///" + os.path.join(tempfile.gettempdir(), "test.db"))
observation = 2.5
gt_par = {'p0': 1, 'p1': 0.25}
abc.new(db_path, {"data": observation}, gt_par=gt_par)
history = abc.history
par_from_history = history.get_ground_truth_parameter()
assert par_from_history == gt_par
def test_r(sampler):
r_file = "doc/examples/myRModel.R"
r = pyabc.external.R(r_file)
r.display_source_ipython()
model = r.model("myModel")
distance = r.distance("myDistance")
sum_stat = r.summary_statistics("mySummaryStatistics")
prior = pyabc.Distribution(meanX=pyabc.RV("uniform", 0, 10),
meanY=pyabc.RV("uniform", 0, 10))
abc = pyabc.ABCSMC(model,
prior,
distance,
summary_statistics=sum_stat,
sampler=sampler)
db = "sqlite:///" + os.path.join(gettempdir(), "test_external.db")
abc.new(db, r.observation("mySumStatData"))
history = abc.run(minimum_epsilon=0.9, max_nr_populations=2)
history.get_weighted_sum_stats_for_model(m=0, t=1)[1][0]["cars"].head()
def test_r():
"""
This is basically just the using_R notebook.
"""
r = R(r_file)
r.display_source_ipython()
model = r.model("myModel")
distance = r.distance("myDistance")
sum_stat = r.summary_statistics("mySummaryStatistics")
prior = pyabc.Distribution(meanX=pyabc.RV("uniform", 0, 10),
meanY=pyabc.RV("uniform", 0, 10))
sampler = pyabc.sampler.MulticoreEvalParallelSampler(n_procs=2)
abc = pyabc.ABCSMC(model, prior, distance,
summary_statistics=sum_stat,
sampler=sampler)
db = "sqlite:///" + os.path.join(gettempdir(), "test_external.db")
abc.new(db, r.observation("mySumStatData"))
history = abc.run(minimum_epsilon=0.9, max_nr_populations=2)
history.get_weighted_sum_stats_for_model(m=0, t=1)[1][0]["cars"].head()
def test_pipeline(db_path):
model = BasicoModel(MODEL1_PATH, duration=MAX_T, method="deterministic")
data = model(TRUE_PAR)
prior = pyabc.Distribution(rate=pyabc.RV("uniform", 0, 100))
n_test_times = 20
t_test_times = np.linspace(0, MAX_T, n_test_times)
def distance(x, y):
xt_ind = np.searchsorted(x["t"], t_test_times) - 1
yt_ind = np.searchsorted(y["t"], t_test_times) - 1
error = (
np.absolute(x["X"][:, 1][xt_ind] - y["X"][:, 1][yt_ind]).sum() /
t_test_times.size)
return error
abc = pyabc.ABCSMC(model, prior, distance)
abc.new(db_path, data)
abc.run(max_nr_populations=3)
def test_stochastic_acceptor():
"""Test the stochastic acceptor's features."""
# store pnorms
pnorm_file = tempfile.mkstemp(suffix=".json")[1]
acceptor = pyabc.StochasticAcceptor(
pdf_norm_method=pyabc.pdf_norm_max_found, log_file=pnorm_file)
eps = pyabc.Temperature(initial_temperature=1)
distance = pyabc.IndependentNormalKernel(var=np.array([1, 1]))
def model(par):
return {'s0': par['p0'] + np.array([0.3, 0.7])}
x_0 = {'s0': np.array([0.4, -0.6])}
# just run
prior = pyabc.Distribution(p0=pyabc.RV('uniform', -1, 2))
abc = pyabc.ABCSMC(model,
prior,
distance,
eps=eps,
acceptor=acceptor,
population_size=10)
abc.new(pyabc.create_sqlite_db_id(), x_0)
h = abc.run(max_nr_populations=1, minimum_epsilon=1.)
# check pnorms
pnorms = pyabc.storage.load_dict_from_json(pnorm_file)
assert len(pnorms) == h.max_t + 2 # +1 t0, +1 one final update
assert isinstance(list(pnorms.keys())[0], int)
assert isinstance(pnorms[0], float)
# use no initial temperature and adaptive c
acceptor = pyabc.StochasticAcceptor()
eps = pyabc.Temperature()
abc = pyabc.ABCSMC(model,
prior,
distance,
eps=eps,
acceptor=acceptor,
population_size=20)
abc.new(pyabc.create_sqlite_db_id(), x_0)
abc.run(max_nr_populations=3)
def test_redis_subprocess():
"""Test whether the instructed redis sampler allows worker subprocesses."""
# print worker output
logging.getLogger("Redis-Worker").addHandler(logging.StreamHandler())
def model_process(p, pipe):
"""The actual model."""
pipe.send({"y": p['p0'] + 0.1 * np.random.randn(10)})
def model(p):
"""Model calling a subprocess."""
parent, child = multiprocessing.Pipe()
proc = multiprocessing.Process(target=model_process, args=(p, child))
proc.start()
res = parent.recv()
proc.join()
return res
prior = pyabc.Distribution(p0=pyabc.RV('uniform', -5, 10),
p1=pyabc.RV('uniform', -2, 2))
def distance(y1, y2):
return np.abs(y1['y'] - y2['y']).sum()
obs = {'y': 1}
# False as daemon argument is ok, True and None are not allowed
sampler = RedisEvalParallelSamplerServerStarter(workers=1,
processes_per_worker=2,
daemon=False)
try:
abc = pyabc.ABCSMC(model,
prior,
distance,
sampler=sampler,
population_size=10)
abc.new(pyabc.create_sqlite_db_id(), obs)
# would just never return if model evaluation fails
abc.run(max_nr_populations=3)
finally:
sampler.shutdown()
def calibrate(observed: dict, hostname: str = None):
"""Calibrates. observed is a dictionary with keys as in calibration_statistic.statistics containing the real data"""
db_path = "sqlite:///" + os.path.join(os.getcwd(), "data",
"calibration.db")
if hostname is not None:
# If we're given a hostname, use the above sandman mapping wrapper
sampler = pyabc.sampler.MappingSampler(map_=get_sm_map(hostname),
mapper_pickles=True)
else:
# Otherwise, run locally with the normal sampler
sampler = pyabc.sampler.MulticoreEvalParallelSampler()
# Adaptive distance based on Prangle (2017) (also acceptor below)
dist = pyabc.distance.AdaptivePNormDistance(p=2, adaptive=True)
prior = pyabc.Distribution(**get_prior())
pop_size = pyabc.populationstrategy.AdaptivePopulationSize(
start_nr_particles=32, max_population_size=256, min_population_size=4)
abc = pyabc.ABCSMC(
model,
parameter_priors=prior,
distance_function=dist,
population_size=pop_size,
sampler=sampler,
acceptor=pyabc.accept_use_complete_history,
)
run_id = abc.new(db=db_path, observed_sum_stat=observed)
print(f"Run ID is {run_id}")
history = abc.run(max_nr_populations=10)
df, w = history.get_distribution()
results = {}
for param in df.columns.values:
# Calculate the posterior mean of each parameter
results[param] = np.dot(list(df[param]), list(w))
print("Done! The results are:")
print(results)
index_col=0,
usecols=["Parameter", "Min", "Range"])
vvwm_ranges = pd.read_csv(os.path.join(master_path,
"vvwm_param_priors.csv"),
index_col=0,
usecols=["Parameter", "Min", "Range"])
param_ranges = pd.concat([swmm_ranges, vvwm_ranges], axis=0)
# take just a tiny subset if in debug mode
if mode == "debug" and debug_params:
param_ranges = param_ranges.iloc[debug_params]
# make the dataframe into a dictionary
priors = param_ranges.to_dict("index")
# make the dictionary into a pyabc distribution object
prior = pyabc.Distribution(
**{
key: pyabc.RV("uniform", loc=v['Min'], scale=v['Range'])
for key, v in priors.items()
})
# import the dictionary of observed data (benchmarks for comparison) depending on the mode
if mode == 'debug':
with open(
os.path.join(main_path, 'master_debug', 'debug_obs_data.txt'),
'r') as read_file:
obs_dict = eval(read_file.read())
elif mode == 'test':
with open(os.path.join(main_path, 'master_test', 'test_obs_data.txt'),
'r') as read_file:
obs_dict = eval(read_file.read())
elif mode == 'run':
with open(os.path.join(main_path, 'master', 'obs_data.txt'),
'r') as read_file:
prior_distribution = "uniform"
print(prior_distribution)
# args = (para["lambda_n"], para["k_n_beta"], para["mu_n"], para["v_n_phi"],
# para["lambda_phi"], para["k_phi_beta"], para["mu_phi"],
# para["s_beta_n"], para["i_beta_phi"], para["mu_beta"],
# para["s_alpha_phi"], para["mu_alpha"])
para_prior1 = pyabc.Distribution(
lambda_n=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
k_n_beta=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
mu_n=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
v_n_phi=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
lambda_phi=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
k_phi_beta=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
mu_phi=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
s_beta_n=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
i_beta_phi=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
mu_beta=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
s_alpha_phi=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
mu_alpha=pyabc.RV(prior_distribution, lim.lb, lim.interval_length))
para_prior2 = pyabc.Distribution(
lambda_n=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
a=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
k_n_beta=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
mu_n=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
v_n_phi=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
k_phi_beta=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
mu_phi=pyabc.RV(prior_distribution, lim.lb, lim.interval_length),
# %% Define prior distribution of parameters
# Be careful that RV("uniform", -10, 15) means uniform distribution in [-10, 5], '15' here is the interval length
lim = PriorLimits(0, 20)
lim2 = PriorLimits(0, 1)
lim3 = PriorLimits(0, 10)
# lim2 = PriorLimits(0, 20)
# lim3 = PriorLimits(0, 20)
paraPrior = pyabc.Distribution(
lambdaN=pyabc.RV("uniform", lim3.lb, lim3.interval_length),
kNB=pyabc.RV("uniform", lim3.lb, lim3.interval_length),
muN=pyabc.RV("uniform", lim2.lb, lim2.interval_length),
vNM=pyabc.RV("uniform", lim2.lb, lim2.interval_length),
lambdaM=pyabc.RV("uniform", lim3.lb, lim3.interval_length),
kMB=pyabc.RV("uniform", lim2.lb, lim2.interval_length),
muM=pyabc.RV("uniform", lim2.lb, lim2.interval_length),
sBN=pyabc.RV("uniform", lim3.lb, lim3.interval_length),
iBM=pyabc.RV("uniform", lim3.lb, lim3.interval_length),
muB=pyabc.RV("uniform", lim3.lb, lim3.interval_length),
sAM=pyabc.RV("uniform", lim.lb, lim.interval_length),
muA=pyabc.RV("uniform", lim.lb, lim.interval_length)
)
# %% Define ABC-SMC model
# distanceP2_adpt = pyabc.AdaptivePNormDistance(p=2,
# scale_function=pyabc.distance.root_mean_square_deviation,
# factors=factors
# )
distanceP2 = pyabc.PNormDistance(p=2, weights=None, factors=factors)
# kernel1 = pyabc.IndependentNormalKernel(var=1.0 ** 2)
def get_prior(self):
return pyabc.Distribution(
**{key: pyabc.RV('norm', 0.5, 0.4)
for key in self.limits})
def get_prior(self):
return pyabc.Distribution(
**{
key: pyabc.RV('uniform', bounds[0], bounds[1])
for key, bounds in self.limits.items()
})
limits["k7"] = (1, 4)
limits["k8"] = (1, 4)
limits["k2"] = (0, 1)
limits["K_M1"] = (-3.6, -2)
# limits_2 = dict()
# limits_2["k3_0"] = (-1, 1)
# limits_2["k4"] = (-1, 1)
# limits_2["k6"] = (-1, 1)
# limits_2["k9"] = (-1, 1)
# limits_2["k10"] = (-1, 1)
# limits_2["k11"] = (-1, 1)
# limits_2["intensity_normalization_total"] = (-1, 1)
prior = pyabc.Distribution(**{
key: pyabc.RV("uniform", lb, ub - lb)
for key, (lb, ub) in limits.items()
})
def eucl_dist_Jan(sim, obs):
total = 0
for key in sim:
if key == 'loc': continue
x = np.array(sim[key])
y = np.array(obs[key])
if np.max(y) != 0:
x = x / np.max(y)
y = y / np.max(y)
total += np.sum((x - y)**2) / x.size
return total