def test_set_params_occupation_network_by_age(self):
p = Parameters(
read_param_file=True,
input_households="tests/data/baseline_household_demographics.csv",
input_param_file="tests/data/baseline_parameters.csv",
param_line_number=1,
)
p.set_param("lockdown_occupation_multiplier_primary_network", 100.0)
assert p.get_param(
"lockdown_occupation_multiplier_primary_network") == 100.0
assert p.get_param(
"lockdown_occupation_multiplier_working_network") == 0.2
def test_set_params_from_python(self):
all_params = pd.read_csv("tests/data/baseline_parameters.csv")
p = Parameters(
read_param_file=False,
input_households="tests/data/baseline_household_demographics.csv")
for key in all_params:
value = all_params[key][0]
try:
p.set_param(key, float(value))
except TypeError:
p.set_param(key, int(value))
assert pytest.approx(p.get_param(key),
value), f"{key} was not set properly"
p._read_household_demographics()
def setup_parameters(d: dict = None):
# Set up Parameters
# Override defaults that we pass in input dict
p = Parameters(
input_param_file=str(BASELINE_PARAMS),
param_line_number=1,
output_file_dir=str(OUTPUT_DIR),
input_households=str(HOUSEHOLDS),
read_param_file=True,
)
if d:
for k, v in d.items():
p.set_param(k, v)
return p
def test_run_model_read_prama_file_false(self):
all_params = pd.read_csv("tests/data/baseline_parameters.csv")
p = Parameters(
read_param_file=False,
input_households="tests/data/baseline_household_demographics.csv",
)
for key in all_params:
value = all_params[key][0]
try:
p.set_param(key, float(value))
except TypeError:
p.set_param(key, int(value))
m = Model(p)
m.one_time_step()
assert m.one_time_step_results()["time"] == 1
def test_set_params_manual_traceable_fraction(self):
p = Parameters(
read_param_file=True,
input_households="tests/data/baseline_household_demographics.csv",
input_param_file="tests/data/baseline_parameters.csv",
param_line_number=1,
)
p.set_param("manual_traceable_fraction_occupation", 0.8)
assert p.get_param("manual_traceable_fraction_occupation") == 0.8
p.set_param("manual_traceable_fraction_household", 0.6)
assert p.get_param("manual_traceable_fraction_household") == 0.6
assert p.get_param("manual_traceable_fraction_occupation") == 0.8
model = Model(p)
model.update_running_params("manual_traceable_fraction_household", 0.4)
assert model.get_param("manual_traceable_fraction_household") == 0.4
def setup_params(updated_params: Dict[str, Any] = None):
"""[summary]
set up a parameters object from the baseline file
Customise any parameters supplied in the updated params dict
Keyword Arguments:
updated_params {dict} -- [description] (default: None)
Returns:
Parameter set
"""
p = Parameters(
input_param_file=os.fspath(BASELINE_PARAMS),
output_file_dir=os.fspath(OUTPUT_DIR),
param_line_number=1,
input_households=os.fspath(HOUSEHOLDS),
read_param_file=True,
)
if updated_params:
for k, v in updated_params.items():
p.set_param(k, v)
return p
from COVID19.model import Model, Parameters
import COVID19.simulation as simulation
import pandas as pd
input_params = pd.read_csv('./tests/data/baseline_parameters.csv')
input_households = pd.read_csv('./tests/data/baseline_household_demographics.csv')
print(input_params)
print(input_households)
params = Parameters(
input_param_file="./tests/data/baseline_parameters.csv",
param_line_number=1,
output_file_dir="./data_test",
input_households="./tests/data/baseline_household_demographics.csv"
)
params.set_param( "n_total", 10000)
params.set_param( "app_turn_on_time", 10)
model = simulation.COVID19IBM(model = Model(params))
sim = simulation.Simulation(env = model, end_time = 100 )
sim.steps( 1000 )
print( pd.DataFrame.from_dict(sim.results) )
def _worker(
args: Tuple[Region, Mapping[Age10Y, float], multiprocessing.Queue],
output_dir: str,
parameters_path: str,
household_demographics_path: str,
total_individuals: int,
lockdown_start: Optional[int],
lockdown_end: int,
end_time: int,
) -> None:
region, populations, queue = args
# Prepare output paths for each region
output_file_name = pathlib.Path(output_dir).joinpath(f"{region.name}_ts.csv")
detailed_output_dir = pathlib.Path(output_dir).joinpath(f"{region.name}_detailed")
os.makedirs(detailed_output_dir, exist_ok=True)
# Init parameters from repo baseline
params = Parameters(
parameters_path, 1, str(detailed_output_dir), household_demographics_path,
)
# Set more simulation params
params.set_param("n_total", total_individuals)
params.set_param("end_time", end_time)
# FIXME: not writing detailed output
params.set_param("sys_write_individual", 1)
# Set populations
for k, v in populations.items():
params.set_param(k.value, v)
# Activate lockdown if specified
if lockdown_start is not None:
for k, v in LOCKDOWN_PARMAETERS.items():
params.set_param(k, v)
params.set_param("lockdown_time_on", lockdown_start)
params.set_param("lockdown_time_off", lockdown_end)
# Set up model
model = simulation.COVID19IBM(model=Model(params))
agent = simulation.Agent()
results = collections.defaultdict(list)
# Set up simulation
current_state = model.start_simulation()
current_action = agent.start_simulation(current_state)
# Run simulation
for step in range(end_time):
current_state = model.step(current_action)
current_action = agent.step(current_state)
for key, value in current_state.items():
results[key].append(value)
queue.put((step, current_state))
# Extract and write outputs
outputs = pd.DataFrame(results)
outputs.drop(columns=["test_on_symptoms", "app_turned_on"])
outputs.to_csv(output_file_name, index=False)
def test_quarantine_totals(self):
"""
Test the totals in the time series output are consistent with the "quarantine reasons" files
"""
T = 100
params = Parameters(constant.TEST_DATA_TEMPLATE, 1,
constant.DATA_DIR_TEST,
constant.TEST_HOUSEHOLD_TEMPLATE,
constant.TEST_HOSPITAL_FILE, 1, True, True)
params.set_param("n_total", 100000)
model = utils.get_model_swig(params)
results = list()
# Run for 30 days
t = 1
while t <= 30:
model.one_time_step()
model.write_quarantine_reasons()
results.append(model.one_time_step_results())
t += 1
# Turn on self-quarantining and run for another 10 days
model.update_running_params("self_quarantine_fraction", 0.8)
model.update_running_params("quarantine_household_on_positive", 1)
model.update_running_params("quarantine_household_on_symptoms", 1)
while t <= 40:
model.one_time_step()
model.write_quarantine_reasons()
results.append(model.one_time_step_results())
t += 1
# Turn on the app and run until time 100
model.update_running_params("app_turned_on", 1)
model.update_running_params("quarantine_on_traced", 1)
while t < T:
model.one_time_step()
model.write_quarantine_reasons()
results.append(model.one_time_step_results())
t += 1
df_ts = pd.DataFrame(results)
# Test that quarantining events (and release events) add to total numbers in quarantine
n_quar_events = np.cumsum(df_ts["n_quarantine_events"].values - \
df_ts["n_quarantine_release_events"].values)
np.testing.assert_array_equal(df_ts["n_quarantine"].values,
n_quar_events)
n_quar_events_app_user = np.cumsum(df_ts["n_quarantine_events_app_user"].values - \
df_ts["n_quarantine_release_events_app_user"].values)
np.testing.assert_array_equal(df_ts["n_quarantine_app_user"].values,
n_quar_events_app_user)
dfs =[pd.read_csv(constant.TEST_QUARANTINE_REASONS_FILE.substitute(T = t)) \
for t in np.arange(1, T)]
dfs = pd.concat(dfs)
dfs["infected"] = (dfs.status >
constant.EVENT_TYPES.SUSCEPTIBLE.value) * 1
dfs["recovered"] = (dfs.status
== constant.EVENT_TYPES.RECOVERED.value) * 1
idx_vars = ["time", "app_user", "infected", "recovered"]
dfs_gp = dfs.groupby(idx_vars)["ID"].count().reset_index()
# Calculate n_quarantine
n_quarantine = dfs_gp.groupby("time")["ID"].sum()
n_quarantine = n_quarantine.reindex(np.arange(1, T)).fillna(0).values
np.testing.assert_array_equal(n_quarantine, df_ts.n_quarantine.values)
# Calculate n_quarantine_app_user
n_quarantine_app_user = dfs_gp.loc[dfs_gp.app_user == 1].groupby(
"time")["ID"].sum()
n_quarantine_app_user = n_quarantine_app_user.reindex(np.arange(
1, T)).fillna(0).values
np.testing.assert_array_equal(n_quarantine_app_user,
df_ts.n_quarantine_app_user.values)
# Calculate n_quarantine_infected
n_quarantine_infected = dfs_gp.loc[dfs_gp.infected == 1].groupby(
"time")["ID"].sum()
n_quarantine_infected = n_quarantine_infected.reindex(np.arange(
1, T)).fillna(0).values
np.testing.assert_array_equal(n_quarantine_infected,
df_ts.n_quarantine_infected.values)
# Calculate n_quarantine_recovered
n_quarantine_recovered = dfs_gp.loc[dfs_gp.recovered == 1].groupby(
"time")["ID"].sum()
n_quarantine_recovered = n_quarantine_recovered.reindex(np.arange(
1, T)).fillna(0).values
# Calculate n_quarantine_app_user_infected
n_quarantine_app_user_infected = \
dfs_gp.loc[(dfs_gp.app_user==1)&(dfs_gp.infected==1)].groupby("time")["ID"].sum()
n_quarantine_app_user_infected = \
n_quarantine_app_user_infected.reindex(np.arange(1, T)).fillna(0).values
np.testing.assert_array_equal(
n_quarantine_app_user_infected,
df_ts.n_quarantine_app_user_infected.values)
# Calculate n_quarantine_app_user_recovered
n_quarantine_app_user_recovered = \
dfs_gp.loc[(dfs_gp.app_user==1)&(dfs_gp.recovered==1)].groupby("time")["ID"].sum()
n_quarantine_app_user_recovered = \
n_quarantine_app_user_recovered.reindex(np.arange(1, T)).fillna(0).values
np.testing.assert_array_equal(
n_quarantine_app_user_recovered,
df_ts.n_quarantine_app_user_recovered.values)
def free_abm(
params,
logger=dummy_logger(),
input_parameter_file="./abm_params/baseline_parameters.csv",
household_demographics_file="./abm_params/baseline_household_demographics.csv",
parameter_line_number=1,
name_file_res="res",
output_dir="./output/",
save_every_iter=5,
stop_zero_I=True,
data={},
callback=lambda data: None):
'''
Simulate the openABM epidemic simulation with no intervention strategy
input
-----
params: Dict
Dictonary with openABM to set
results:
print on file true configurations and transmission
'''
params_model = Parameters(input_parameter_file, parameter_line_number,
output_dir, household_demographics_file)
fold_out = Path(output_dir)
if not fold_out.exists():
fold_out.mkdir(parents=True)
params_model = Parameters()
for k, val in params.items():
params_model.set_param(k, val)
model = simulation.COVID19IBM(model=Model(params_model))
T = params_model.get_param("end_time")
N = params_model.get_param("n_total")
sim = simulation.Simulation(env=model, end_time=T, verbose=False)
for col_name in ["I", "IR"]:
data[col_name] = np.full(T, np.nan)
for t in range(T):
sim.steps(1)
status = np.array(covid19.get_state(model.model.c_model))
state = status_to_state(status)
nS, nI, nR = (state == 0).sum(), (state == 1).sum(), (state == 2).sum()
if nI == 0 and stop_zero_I:
logger.info(
"stopping simulation as there are no more infected individuals"
)
break
logger.info(f'time:{t}')
data["I"][t] = nI
data["IR"][t] = nI + nR
callback(data)
# print and import files
sim.env.model.write_individual_file()
df_indiv = pd.read_csv(output_dir + "individual_file_Run1.csv",
skipinitialspace=True)
df_indiv.to_csv(output_dir + name_file_res + "_individuals.gz")
sim.env.model.write_transmissions()
df_trans = pd.read_csv(output_dir + "transmission_Run1.csv")
df_trans.to_csv(output_dir + name_file_res + "_transmissions.gz")
print("End of Simulation")
return
def loop_abm(
params,
inference_algo,
logger=dummy_logger(),
input_parameter_file="./abm_params/baseline_parameters.csv",
household_demographics_file="./abm_params/baseline_household_demographics.csv",
parameter_line_number=1,
seed=1,
initial_steps=0,
num_test_random=50,
num_test_algo=50,
fraction_SM_obs=0.2,
fraction_SS_obs=1,
quarantine_HH=False,
test_HH=False,
name_file_res="res",
output_dir="./output/",
save_every_iter=5,
stop_zero_I=True,
adoption_fraction=1.0,
fp_rate=0.0,
fn_rate=0.0,
smartphone_users_abm=False, # if True use app users fraction from OpenABM model
callback=lambda x: None,
data={}):
'''
Simulate interventions strategy on the openABM epidemic simulation.
input
-----
params: Dict
Dictonary with openABM to set
inference_algo: Class (rank_template)
Class for order the nodes according to the prob to be infected
logger = logger for printing intermediate steps
results:
print on file true configurations and transmission
'''
params_model = Parameters(input_parameter_file, parameter_line_number,
output_dir, household_demographics_file)
### create output_dir if missing
fold_out = Path(output_dir)
if not fold_out.exists():
fold_out.mkdir(parents=True)
### initialize a separate random stream
rng = np.random.RandomState()
rng.seed(seed)
### initialize ABM model
for k, val in params.items():
params_model.set_param(k, val)
model = Model(params_model)
model = simulation.COVID19IBM(model=model)
T = params_model.get_param("end_time")
N = params_model.get_param("n_total")
sim = simulation.Simulation(env=model, end_time=T, verbose=False)
house = covid19.get_house(model.model.c_model)
housedict = listofhouses(house)
has_app = covid19.get_app_users(
model.model.c_model) if smartphone_users_abm else np.ones(N, dtype=int)
has_app &= (rng.random(N) <= adoption_fraction)
### init data and data_states
data_states = {}
data_states["true_conf"] = np.zeros((T, N))
data_states["statuses"] = np.zeros((T, N))
data_states["tested_algo"] = []
data_states["tested_random"] = []
data_states["tested_SS"] = []
data_states["tested_SM"] = []
for name in [
"num_quarantined", "q_SS", "q_SM", "q_algo", "q_random", "q_all",
"infected_free", "S", "I", "R", "IR", "aurI", "prec1%", "prec5%",
"test_+", "test_-", "test_f+", "test_f-"
]:
data[name] = np.full(T, np.nan)
data["logger"] = logger
### init inference algo
inference_algo.init(N, T)
### running variables
indices = np.arange(N, dtype=int)
excluded = np.zeros(N, dtype=bool)
daily_obs = []
all_obs = []
all_quarantined = []
freebirds = 0
num_quarantined = 0
fp_num = 0
fn_num = 0
p_num = 0
n_num = 0
noise_SM = rng.random(N)
noise_SS = rng.random(N) if fraction_SS_obs < 1 else None
nfree = params_model.get_param("n_seed_infection")
for t in range(T):
### advance one time step
sim.steps(1)
status = np.array(covid19.get_state(model.model.c_model))
state = status_to_state(status)
data_states["true_conf"][t] = state
nS, nI, nR = (state == 0).sum(), (state == 1).sum(), (state == 2).sum()
if nI == 0 and stop_zero_I:
logger.info(
"stopping simulation as there are no more infected individuals"
)
break
if t == initial_steps:
logger.info("\nobservation-based inference algorithm starts now\n")
logger.info(f'time:{t}')
### extract contacts
daily_contacts = covid19.get_contacts_daily(model.model.c_model, t)
logger.info(f"number of unique contacts: {len(daily_contacts)}")
### compute potential test results for all
if fp_rate or fn_rate:
noise = rng.random(N)
f_state = (state == 1) * (noise > fn_rate) + (state == 0) * (
noise < fp_rate) + 2 * (state == 2)
else:
f_state = state
to_quarantine = []
all_test = []
excluded_now = excluded.copy()
fp_num_today = 0
fn_num_today = 0
p_num_today = 0
n_num_today = 0
def test_and_quarantine(rank, num):
nonlocal to_quarantine, excluded_now, all_test, fp_num_today, fn_num_today, p_num_today, n_num_today
test_rank = []
for i in rank:
if len(test_rank) == num:
break
if excluded_now[i]:
continue
test_rank += [i]
if f_state[i] == 1:
p_num_today += 1
if state[i] != 1:
fp_num_today += 1
q = housedict[house[i]] if quarantine_HH else [i]
excluded_now[q] = True
to_quarantine += q
excluded[q] = True
if test_HH:
all_test += q
else:
all_test += [i]
else:
n_num_today += 1
if state[i] == 1:
fn_num_today += 1
excluded_now[i] = True
all_test += [i]
return test_rank
### compute rank from algorithm
num_test_algo_today = num_test_algo
if t < initial_steps:
daily_obs = []
num_test_algo_today = 0
weighted_contacts = [(c[0], c[1], c[2], 2.0 if c[3] == 0 else 1.0)
for c in daily_contacts
if (has_app[c[0]] and has_app[c[1]])]
if nfree == 0 and quarantine_HH:
print("faster end")
rank_algo = np.zeros((N, 2))
rank_algo[:, 0] = np.arange(N)
rank_algo[:, 1] = np.random.rand(N)
else:
rank_algo = inference_algo.rank(t, weighted_contacts, daily_obs,
data)
rank = np.array(sorted(rank_algo, key=lambda tup: tup[1],
reverse=True))
rank = [int(tup[0]) for tup in rank]
### test num_test_algo_today individuals
test_algo = test_and_quarantine(rank, num_test_algo_today)
### compute roc now, only excluding past tests
eventsI = events_list(t, [(i, 1, t)
for (i, tf) in enumerate(excluded) if tf],
data_states["true_conf"],
check_fn=check_fn_I)
xI, yI, aurI, sortlI = roc_curve(dict(rank_algo), eventsI, lambda x: x)
### test all SS
if fraction_SS_obs < 1:
SS = indices[(status == 4) & (noise_SS < fraction_SS_obs)]
else:
SS = indices[status == 4]
SS = test_and_quarantine(SS, len(SS))
### test a fraction of SM
SM = indices[(status == 5) & (noise_SM < fraction_SM_obs)]
SM = test_and_quarantine(SM, len(SM))
### do num_test_random extra random tests
test_random = test_and_quarantine(rng.permutation(N), num_test_random)
### quarantine infected individuals
num_quarantined += len(to_quarantine)
covid19.intervention_quarantine_list(model.model.c_model,
to_quarantine, T + 1)
### update observations
daily_obs = [(int(i), int(f_state[i]), int(t)) for i in all_test]
all_obs += daily_obs
### exclude forever nodes that are observed recovered
rec = [i[0] for i in daily_obs if f_state[i[0]] == 2]
excluded[rec] = True
### update data
data_states["tested_algo"].append(test_algo)
data_states["tested_random"].append(test_random)
data_states["tested_SS"].append(SS)
data_states["tested_SM"].append(SM)
data_states["statuses"][t] = status
data["S"][t] = nS
data["I"][t] = nI
data["R"][t] = nR
data["IR"][t] = nR + nI
data["aurI"][t] = aurI
prec = lambda f: yI[int(f / 100 * len(yI))] / int(f / 100 * len(
yI)) if len(yI) else np.nan
ninfq = sum(state[to_quarantine] > 0)
nfree = int(nI - sum(excluded[state == 1]))
data["aurI"][t] = aurI
data["prec1%"][t] = prec(1)
data["prec5%"][t] = prec(5)
data["num_quarantined"][t] = num_quarantined
data["test_+"][t] = p_num
data["test_-"][t] = n_num
data["test_f+"][t] = fp_num
data["test_f-"][t] = fn_num
data["q_SS"][t] = len(SS)
data["q_SM"][t] = len(SM)
sus_test_algo = sum(state[test_algo] == 0)
inf_test_algo = sum(state[test_algo] == 1)
rec_test_algo = sum(state[test_algo] == 2)
inf_test_random = sum(state[test_random] == 1)
data["q_algo"][t] = inf_test_algo
data["q_random"][t] = sum(state[test_random] == 1)
data["infected_free"][t] = nfree
asbirds = 'a bird' if nfree == 1 else 'birds'
fp_num += fp_num_today
fn_num += fn_num_today
n_num += n_num_today
p_num += p_num_today
### show output
logger.info(f"True : (S,I,R): ({nS:.1f}, {nI:.1f}, {nR:.1f})")
logger.info(
f"AUR_I : {aurI:.3f}, prec(1% of {len(yI)}): {prec(1):.2f}, prec5%: {prec(5):.2f}"
)
logger.info(
f"SS: {len(SS)}, SM: {len(SM)}, results test algo (S,I,R): ({sus_test_algo},{inf_test_algo},{rec_test_algo}), infected test random: {inf_test_random}/{num_test_random}"
)
logger.info(
f"false+: {fp_num} (+{fp_num_today}), false-: {fn_num} (+{fn_num_today})"
)
logger.info(
f"...quarantining {len(to_quarantine)} guys -> got {ninfq} infected, {nfree} free as {asbirds} ({nfree-freebirds:+d})"
)
freebirds = nfree
### callback
callback(data)
if t % save_every_iter == 0:
df_save = pd.DataFrame.from_records(data, exclude=["logger"])
df_save.to_csv(output_dir + name_file_res + "_res.gz")
# save files
df_save = pd.DataFrame.from_records(data, exclude=["logger"])
df_save.to_csv(output_dir + name_file_res + "_res.gz")
with open(output_dir + name_file_res + "_states.pkl",
mode="wb") as f_states:
pickle.dump(data_states, f_states)
sim.env.model.write_individual_file()
df_indiv = pd.read_csv(output_dir + "individual_file_Run1.csv",
skipinitialspace=True)
df_indiv.to_csv(output_dir + name_file_res + "_individuals.gz")
sim.env.model.write_transmissions()
df_trans = pd.read_csv(output_dir + "transmission_Run1.csv")
df_trans.to_csv(output_dir + name_file_res + "_transmissions.gz")
return df_save