def abc_setup():
"""
Create abc model
"""
abc_prior_dict = {
's': RV("uniform", *simtools.PARAMS['abc_limits_shape']),
'c': RV("uniform", *simtools.PARAMS['abc_limits_center']),
'w': RV("uniform", *simtools.PARAMS['abc_limits_width']),
'n': RV("uniform", *simtools.PARAMS['abc_limits_noise_sigma']),
'm': RV("uniform", *simtools.PARAMS['abc_limits_normal_maximum_rate']),
'r': RV("uniform", *simtools.PARAMS['abc_limits_maximum_rate_ratio']),
}
abc_priors = Distribution(abc_prior_dict)
abc = ABCSMC(abc_model,
abc_priors,
abc_distance,
population_size=AdaptivePopulationSize(
simtools.PARAMS['abc_initial_population_size'],
mean_cv=simtools.PARAMS['abc_population_size_epsilon'],
max_population_size=simtools.
PARAMS['abc_population_size_maximum']),
sampler=MulticoreEvalParallelSampler(
simtools.PARAMS['abc_parallel_simulations']))
return abc
def test_stop_early(db_path):
mc_sampler = MulticoreEvalParallelSampler(check_max_eval=True)
sc_sampler = SingleCoreSampler(check_max_eval=True)
for sampler in [mc_sampler, sc_sampler]:
abc = ABCSMC(model, Distribution(par=st.uniform(0, 10)), dist, 10,
sampler=sampler)
abc.new(db_path, {"par": .5})
history = abc.run(
max_nr_populations=8, min_acceptance_rate=set_acc_rate)
df = history.get_all_populations()
df["acceptance_rate"] = df["particles"] / df["samples"]
assert df["acceptance_rate"].iloc[-1] >= set_acc_rate
def abc_setup(birthrate_groups):
"""
create abc model
parameters are stored in the global simtools.PARAMS dict
"""
for curve_resolution in simtools.PARAMS['abc_params']['resolution_limits']:
assert curve_resolution > 0 and curve_resolution <= 9
abc_priors = []
for resolution_limit in range(
simtools.PARAMS['abc_params']['resolution_limits'][0],
simtools.PARAMS['abc_params']['resolution_limits'][1] + 1):
abc_prior_dict = {}
for i in range(resolution_limit):
abc_prior_dict['r' + str(i)] = \
RV("uniform", simtools.PARAMS['abc_params']['rate_limits'][0],
abs(simtools.PARAMS['abc_params']['rate_limits'][1] - \
simtools.PARAMS['abc_params']['rate_limits'][0]))
abc_priors.append(
Distribution(birthrate=copy.deepcopy(abc_prior_dict)))
print('priors', abc_priors)
#abc = ABCSMC([abc_model for __ in abc_priors], abc_priors, abc_distance,
# population_size=AdaptivePopulationSize(
# int(simtools.PARAMS['abc_params']['starting_population_size']),
# 0.15,
# max_population_size=int(simtools.PARAMS['abc_params']['max_population_size']),
# min_population_size=int(simtools.PARAMS['abc_params']['min_population_size'])),
# sampler=MulticoreEvalParallelSampler(
# simtools.PARAMS['abc_params']['parallel_simulations']))
abc = ABCSMC(
[abc_model for __ in abc_priors],
abc_priors,
abc_distance,
population_size=ConstantPopulationSize(
int(simtools.PARAMS['abc_params']['starting_population_size'])),
sampler=MulticoreEvalParallelSampler(
simtools.PARAMS['abc_params']['parallel_simulations']))
return abc
def test_stop_early(db_path):
"""Test early stopping inside a generation."""
mc_sampler = MulticoreEvalParallelSampler(check_max_eval=True)
sc_sampler = SingleCoreSampler(check_max_eval=True)
for sampler in [mc_sampler, sc_sampler]:
abc = ABCSMC(model,
Distribution(par=st.uniform(0, 10)),
dist,
pop_size,
sampler=sampler)
abc.new(db_path, {"par": .5})
history = abc.run(min_acceptance_rate=set_acc_rate)
df = history.get_all_populations()
# offset with n_procs as more processes can have run at termination
n_procs = sampler.n_procs if hasattr(sampler, 'n_procs') else 1
df["corrected_acceptance_rate"] = \
df["particles"] / (df["samples"] - (n_procs-1))
assert df["corrected_acceptance_rate"].iloc[-1] >= set_acc_rate
from scipy import stats
from tempfile import gettempdir
from pyabc import MedianEpsilon, \
LocalTransition, Distribution, RV, ABCSMC, sge, \
AdaptivePNormDistance, PNormDistance, UniformAcceptor
from pyabc.sampler import MulticoreEvalParallelSampler
import time
# Set version number each iteration
temp_folder = "jitAdaptivePNormDistance_2300_t=1_nonunifacc_eps09"
version_number = temp_folder + str(time.time())
# SMCABC parameters:
SMCABC_distance = AdaptivePNormDistance(p=2)
SMCABC_population_size = 30
SMCABC_sampler = MulticoreEvalParallelSampler(40)
SMCABC_transitions = LocalTransition(k_fraction=.5)
SMCABC_eps = MedianEpsilon(500, median_multiplier=0.9)
smcabc_minimum_epsilon = 0.001
smcabc_max_nr_populations = 4
smcabc_min_acceptance_rate = SMCABC_population_size / 25000
# Fixed Parameters
div_path = 1000
L = 2300 # time horizon
p_0 = 238.745 * div_path # initial price
MCSteps = 10**5 # MC steps to generate variance
N_A = 125 # no. market makers = no. liquidity providers
# # True price trajectory
# delta_true = 0.0250 # limit order cancellation rate
'initial_lesion_density', 'sensitive_lesion_proportion',
'starting_mutant_proportion', 'k'
]
priors = {}
for p, b in zip(param_order, BOUNDS):
priors[p] = RV("uniform", b[0], b[1] - b[0])
priors = Distribution(priors)
NUM_CORES = 3
POPULATION_SIZE = 10000
DB_PATH = "full_model_abc.db"
###### ABC ######
distance = PNormDistance(p=1)
sampler = MulticoreEvalParallelSampler(n_procs=NUM_CORES)
abc = ABCSMC(run_model, priors, distance, population_size=POPULATION_SIZE)
db_path = ("sqlite:///" + DB_PATH)
abc.new(db_path, {'distance': 0})
history = abc.run(minimum_epsilon=0.001, max_nr_populations=25)
# The database is stored and can be reloaded if needed - see pyabc documentation
# Here just print the median and 95% credible intervals.
import pyabc.visualization.credible as credible
def get_estimate_and_CI_for_param(param, df, w, confidence=0.95):
vals = np.array(df[param])
def PicklingMulticoreEvalParallelSampler():
return MulticoreEvalParallelSampler(pickle=True)
LAMBDA0_TRUE = 100 # initial order placement depth
C_LAMBDA_TRUE = 10 # limit order placement depth coefficient
DELTA_S_TRUE = 0.0010 # mean reversion strength parameter
# prior range
DELTA_MIN, DELTA_MAX = 0, 0.05
MU_MIN, MU_MAX = 0, 0.05
ALPHA_MIN, ALPHA_MAX = 0.05, 0.5
LAMBDA0_MIN, LAMBDA0_MAX = 50, 300
C_LAMBDA_MIN, C_LAMBDA_MAX = 1, 50
DELTAS_MIN, DELTAS_MAX = 0, 0.005
# Fixed Parameters
PRICE_PATH_DIVIDER = 100
TIME_HORIZON = 3200 # time horizon
P_0 = 238.745 * PRICE_PATH_DIVIDER # initial price
MC_STEPS = 10**5 # MC steps to generate variance
N_A = 125 # no. market makers = no. liquidity providers
# SMCABC parameters:
SMCABC_DISTANCE = AdaptivePNormDistance(
p=2, scale_function=pyabc.distance.root_mean_square_deviation)
SMCABC_POPULATION_SIZE = 30
SMCABC_SAMPLER = MulticoreEvalParallelSampler(ncores)
SMCABC_TRANSITIONS = MultivariateNormalTransition()
SMCABC_EPS = MedianEpsilon(0.01)
SMCABC_ACCEPTOR = UniformAcceptor(use_complete_history=True)
smcabc_minimum_epsilon = 0.0001
smcabc_max_nr_populations = 6
smcabc_min_acceptance_rate = SMCABC_POPULATION_SIZE / 25000
# 210113: using testODE results as "data"
result = outbreak(tstRunParam)
noisedResult = addNoise(result, 20)
noiseFile = dataDir + 'testODE-noise.csv'
rptResult(noisedResult, noiseFile, addDate=True)
ndataDF = pd.DataFrame(noisedResult)
ndataDatedDF = addDateSeries(ndataDF)
# Use default MulticoreEvalParallelSampler sampler
# pool = ThreadPoolExecutor(max_workers=nworkers)
# sampler = ConcurrentFutureSampler(pool)
sampler = MulticoreEvalParallelSampler(ncore)
bnds0 = dict(n_infectious=(10, 1000), beta=(1e-3, 1e-1))
doPlots = True
exptList = []
exptIdx = 0
print('main: TESTING one experiment 3_3')
for maxPop in [3]: #[3,5,10,20,40]:
for maxNR in [3]: # [3,5,10,20,40]:
expt = {'maxABCPop': maxPop, 'maxNRPop': maxNR, 'bounds': bnds0}
expt['idx'] = exptIdx
exptIdx += 1
expt['name'] = f'{maxPop:02}_{maxNR:02}'
exptList.append(expt)
def calibrate_ABC(abcDB):
'''calibrate parameters using pyABC library
'''
# print('TESTING: ncore=4 for hancock')
ncore = n_workers # 4
# sampler = MulticoreParticleParallelSampler(ncore) # DYN sampling strategy
# abcSampler = 'part//'
sampler = MulticoreEvalParallelSampler(
ncore, check_max_eval=True) # STAT sampling strategy
abcSampler = 'eval//'
maxABCPop = ncore
maxNGen = 50
maxABCWallTime = datetime.timedelta(minutes=10)
MinABCAccept = 0.2
MinABCEps = 100
# print('TESTING: minEps=200')
# MinABCEps = 100
# NB: ASSUME > MinABCAccept are accepted each gen
# print('TESTING: limit abc evals=1000')
# maxABCEval = 1000 # maxNGen * ncore / MinABCAccept
maxABCEval = maxNGen * ncore / MinABCAccept / 2
print(
f'calibrate_ABC: {abcSampler} maxNGen={maxNGen} maxABCEval={maxABCEval} maxABCWallTime={maxABCWallTime} MinABCAccept={MinABCAccept} MinABCEps={MinABCEps}'
)
bounds, pkeys = get_bounds()
print(f'calibrate_ABC: bounds={bounds}')
# NB: pyabc.Distribution inherits from scipy.stats the convention of the two argument being
# lowerBound and RANGE (vs. lower bound, upper bound)
iiRange = bounds['ub'][0] - bounds['lb'][0]
betaRange = bounds['ub'][1] - bounds['lb'][1]
if UseTestRate == 'srch':
testRateRange = bounds['ub'][1] - bounds['lb'][2]
cvPrior = pyabc.Distribution(pop_infected=pyabc.RV("uniform",
bounds['lb'][0],
iiRange),
beta=pyabc.RV("uniform", bounds['lb'][1],
betaRange))
# NB: treat cvPrior just like a dict even tho its a ParameterStructure
if UseTestRate == 'srch':
cvPrior['test_rate'] = pyabc.RV("uniform", bounds['lb'][2],
testRateRange)
abc = pyabc.ABCSMC(
cvModel,
cvPrior,
None, # 210222: Use default PNorm()
# population_size=AdaptivePopulationSize(ncore, 0.15,max_population_size=4*ncore)
population_size=maxABCPop,
sampler=sampler)
# No observed mismatch!
history = abc.new(abcDB, {"mismatch": 0})
runID = history.id
print(f'calibrate_ABC: runID={runID}')
start_time = time()
history = abc.run(min_acceptance_rate=MinABCAccept,
minimum_epsilon=MinABCEps,
max_total_nr_simulations=maxABCEval,
max_walltime=maxABCWallTime,
max_nr_populations=50)
print(f"calibrate_ABC: runID={runID} ABC time {time() - start_time:.2f}s")
return history