def test_iterator():
"""This test repeats main_RepeatsNcov.c and validates that the
various stages report the same results as the original C code
for ncov, when using a custom integrator that just calls
iterate_weekday
"""
prompt = None
# user input parameters
seed = 15324
inputfile = ncovparams_csv
line_num = 0
UV = 1.0
# load all of the parameters
try:
params = Parameters.load(parameters="march29")
except Exception as e:
print(f"Unable to load parameter files. Make sure that you have "
f"cloned the MetaWardsData repository and have set the "
f"environment variable METAWARDSDATA to point to the "
f"local directory containing the repository, e.g. the "
f"default is $HOME/GitHub/MetaWardsData")
raise e
# load the disease and starting-point input files
params.set_disease("ncov")
params.set_input_files("2011Data")
params.add_seeds("ExtraSeedsBrighton.dat")
# start from the parameters in the specified line number of the
# provided input file
variables = params.read_variables(inputfile, line_num)
# extra parameters that are set
params.UV = UV
params.static_play_at_home = 0
params.play_to_work = 0
params.work_to_play = 0
params.daily_imports = 0.0
# the size of the starting population
population = Population(initial=57104043)
profiler = Profiler()
print("Building the network...")
network = Network.build(params=params, profiler=profiler)
params = params.set_variables(variables[0])
network.update(params, profiler=profiler)
# Here is a custom integrator function that just calls
# iterate_weekday after 'print_hello'
def print_hello(**kwargs):
print(f"Hello")
def my_iterator(**kwargs):
from metawards.iterators import iterate_weekday
return [print_hello] + iterate_weekday(**kwargs)
from metawards.iterators import build_custom_iterator
iterator = build_custom_iterator(my_iterator, __name__)
print("Run the model...")
outdir = os.path.join(script_dir, "test_integrator_output")
with OutputFiles(outdir, force_empty=True, prompt=prompt) as output_dir:
trajectory = network.run(population=population,
seed=seed,
output_dir=output_dir,
nsteps=29,
profiler=profiler,
iterator=iterator,
nthreads=1)
OutputFiles.remove(outdir, prompt=None)
print("End of the run")
print(f"Model output: {trajectory}")
print(profiler)
# The original C code has this expected population after 47 steps
expected = Population(initial=57104043,
susceptibles=56081730,
latent=134,
total=48,
recovereds=165,
n_inf_wards=28,
day=29)
print(f"Expect output: {expected}")
assert trajectory[-1] == expected
def test_too_small(prompt=None):
"""This test repeats main_RepeatsNcov.c and validates that the
various stages report the same results as the original C code
for ncov
"""
# user input parameters
seed = 15324
inputfile = ncovparams_csv
line_num = 0
UV = 0.0
# load all of the parameters
try:
params = Parameters.load(parameters="march29")
except Exception as e:
print(f"Unable to load parameter files. Make sure that you have "
f"cloned the MetaWardsData repository and have set the "
f"environment variable METAWARDSDATA to point to the "
f"local directory containing the repository, e.g. the "
f"default is $HOME/GitHub/MetaWardsData")
raise e
# load the disease and starting-point input files
params.set_disease(os.path.join(script_dir, "data", "ncov.json"))
params.set_input_files("2011Data")
params.add_seeds("ExtraSeedsBrighton.dat")
# start from the parameters in the specified line number of the
# provided input file
variables = params.read_variables(inputfile, line_num)
# extra parameters that are set
params.UV = UV
params.static_play_at_home = 0
params.play_to_work = 0
params.work_to_play = 0
params.daily_imports = 0.0
# the size of the starting population
population = Population(initial=57104043)
profiler = Profiler()
profiler2 = Profiler()
print("Building the network...")
network = Network.build(params=params,
profiler=profiler,
max_nodes=100,
max_links=100)
print("Building it again... (should be cached)")
network = Network.build(params=params,
profiler=profiler2,
max_nodes=100,
max_links=100)
params = params.set_variables(variables[0])
network.update(params, profiler=None)
print("Run the model...")
outdir = os.path.join(script_dir, "test_integration_output")
with OutputFiles(outdir, force_empty=True, prompt=prompt) as output_dir:
trajectory = network.run(population=population,
seed=seed,
output_dir=output_dir,
nsteps=30,
profiler=None,
nthreads=1)
OutputFiles.remove(outdir, prompt=None)
print("End of the run")
print(profiler)
print(f"Model output: {trajectory}")
expected = Population(initial=57104043,
susceptibles=56081610,
latent=167,
total=60,
recovereds=240,
n_inf_wards=56,
day=30)
print(f"Expect output: {expected}")
assert trajectory[-1] == expected
def test_local():
prompt = None
# user input parameters
seed = 15324
UV = 1.0
# load all of the parameters
try:
params = Parameters.load(parameters="march29")
except Exception as e:
print(f"Unable to load parameter files. Make sure that you have "
f"cloned the MetaWardsData repository and have set the "
f"environment variable METAWARDSDATA to point to the "
f"local directory containing the repository, e.g. the "
f"default is $HOME/GitHub/MetaWardsData")
raise e
# load the disease and starting-point input files
params.set_disease("ncov")
params.set_input_files("2011Data")
params.add_seeds("1 5 2124") # seeding 5 into ward 2124
# extra parameters that are set
params.UV = UV
params.static_play_at_home = 0
params.play_to_work = 0
params.work_to_play = 0
params.daily_imports = 0.0
# the size of the starting population
population = Population(initial=57104043)
print("Building the network...")
network = Network.build(params=params)
outdir = os.path.join(script_dir, "test_local_output")
# First check that setting the local cutoff has the same effect
# as setting the global cutoff
with OutputFiles(outdir, force_empty=True, prompt=prompt) as output_dir:
trajectory = network.copy().run(population=population,
seed=seed,
output_dir=output_dir,
nsteps=50,
iterator=iterate_cutoff,
extractor=extract_cutoff,
nthreads=1)
OutputFiles.remove(outdir, prompt=None)
# run setting the global dyn_dist_cutoff to zero
with OutputFiles(outdir, force_empty=True, prompt=prompt) as output_dir:
net2 = network.copy()
net2.params.dyn_dist_cutoff = 0.0
trajectory2 = net2.run(population=population,
seed=seed,
output_dir=output_dir,
nsteps=50,
extractor=extract_cutoff,
nthreads=1)
OutputFiles.remove(outdir, prompt=None)
print(f"Model output: {trajectory}")
print(f"Model output: {trajectory2}")
assert trajectory == trajectory2
# now test that setting the scale_uv has the same effect as
# setting the global scale_uv
with OutputFiles(outdir, force_empty=True, prompt=prompt) as output_dir:
trajectory = network.copy().run(population=population,
seed=seed,
output_dir=output_dir,
nsteps=50,
iterator=iterate_scale,
extractor=extract_scale,
nthreads=1)
OutputFiles.remove(outdir, prompt=None)
# run setting the global dyn_dist_cutoff to zero
with OutputFiles(outdir, force_empty=True, prompt=prompt) as output_dir:
net2 = network.copy()
pop2 = deepcopy(population)
pop2.scale_uv = 0.0
trajectory2 = net2.run(population=pop2,
seed=seed,
output_dir=output_dir,
nsteps=50,
extractor=extract_scale,
nthreads=1)
OutputFiles.remove(outdir, prompt=None)
print(f"Model output: {trajectory}")
print(f"Model output: {trajectory2}")
p = trajectory[-1]
p2 = trajectory2[-1]
# won't be identical as different scale_uv causes different order
# of random numbers - but should still have 0 infections
assert p.has_equal_SEIR(p2)
# now test that setting both to non-zero values has the same effect
with OutputFiles(outdir, force_empty=True, prompt=prompt) as output_dir:
trajectory = network.copy().run(population=population,
seed=seed,
output_dir=output_dir,
nsteps=50,
iterator=iterate_both,
nthreads=1)
OutputFiles.remove(outdir, prompt=None)
# run setting the global dyn_dist_cutoff to zero
with OutputFiles(outdir, force_empty=True, prompt=prompt) as output_dir:
net2 = network.copy()
net2.params.dyn_dist_cutoff = 42.0
pop2 = deepcopy(population)
pop2.scale_uv = 0.5
trajectory2 = net2.run(population=pop2,
seed=seed,
output_dir=output_dir,
nsteps=50,
nthreads=1)
OutputFiles.remove(outdir, prompt=None)
print(f"Model output: {trajectory}")
print(f"Model output: {trajectory2}")
p = trajectory[-1]
p2 = trajectory2[-1]
p2.scale_uv = 1
assert p == p2
def test_openfiles(prompt=None):
outdir = os.path.join(script_dir, "test_openfiles_output")
if os.path.exists(outdir):
OutputFiles.remove(outdir, prompt=prompt)
of = OutputFiles(outdir)
assert of.is_open()
assert not of.is_closed()
with pytest.raises(ValueError):
of.open("../test.txt")
with pytest.raises(ValueError):
of.open("/test.txt")
FILE = of.open("test.txt")
assert of.is_open()
FILE.write("hello\n")
of.close()
assert of.is_closed()
assert open(os.path.join(outdir, "test.txt")).readline() == "hello\n"
with pytest.raises(FileExistsError):
OutputFiles(outdir, prompt=None)
with OutputFiles(outdir, force_empty=True, prompt=None) as of:
FILE = of.open("test.txt")
assert of.is_open()
assert of.get_filename("test.txt").endswith("test.txt")
FILE.write("goodbye\n")
FILE = of.open("test2.txt", auto_bzip=True)
FILE.write("hello ")
assert of.get_filename("test2.txt").endswith("test2.txt.bz2")
FILE = of.open("test2.txt")
FILE.write("world\n")
assert open(os.path.join(outdir, "test.txt")).readline() == "goodbye\n"
import bz2
line = bz2.open(os.path.join(outdir, "test2.txt.bz2"), "rt").readline()
assert line == "hello world\n"
OutputFiles.remove(outdir, prompt=None)
def test_network_copy(prompt=None, nthreads=1):
# user input parameters
import random
seed = random.randint(100000, 1000000)
UV = 0.0
# load all of the parameters
try:
params = Parameters.load(parameters="march29")
except Exception as e:
print(f"Unable to load parameter files. Make sure that you have "
f"cloned the MetaWardsData repository and have set the "
f"environment variable METAWARDSDATA to point to the "
f"local directory containing the repository, e.g. the "
f"default is $HOME/GitHub/MetaWardsData")
raise e
# load the disease and starting-point input files
params.set_disease(os.path.join(script_dir, "data", "ncov.json"))
params.set_input_files("2011Data")
params.add_seeds("ExtraSeedsBrighton.dat")
# extra parameters that are set
params.UV = UV
params.static_play_at_home = 0
params.play_to_work = 0
params.work_to_play = 0
params.daily_imports = 0.0
# the size of the starting population
population = Population(initial=57104043)
nsteps = 20
print("Building the network...")
network = Network.build(params=params)
outdir = os.path.join(script_dir, "test_network_copy")
print("Run 1")
with OutputFiles(outdir, force_empty=True, prompt=prompt) as output_dir:
t1 = network.copy().run(population=population,
seed=seed,
output_dir=output_dir,
nsteps=nsteps,
extractor=extract_none,
nthreads=nthreads)
OutputFiles.remove(outdir, prompt=None)
print("Run 2")
with OutputFiles(outdir, force_empty=True, prompt=prompt) as output_dir:
t2 = network.copy().run(population=population,
seed=seed,
output_dir=output_dir,
nsteps=nsteps,
extractor=extract_none,
nthreads=nthreads)
print("Run 3")
with OutputFiles(outdir, force_empty=True, prompt=prompt) as output_dir:
t3 = network.copy().run(population=population,
seed=seed,
output_dir=output_dir,
nsteps=nsteps,
extractor=extract_none,
nthreads=nthreads)
OutputFiles.remove(outdir, prompt=None)
print(t1)
print(t2)
print(t3)
assert t1 == t2
assert t1 == t3
def test_pathway():
demographics = Demographics.load(demographics_json)
assert len(demographics) == 2
disease_home = Disease.load(filename=home_json)
disease_super = Disease.load(filename=super_json)
assert demographics[1].disease is None
assert demographics[0].disease == disease_super
params = Parameters.load()
params.set_disease(disease_home)
params.set_input_files("single")
params.add_seeds("ExtraSeedsOne.dat")
network = Network.build(params)
print(network.params.disease_params)
print(disease_home)
assert network.params.disease_params == disease_home
network = network.specialise(demographics)
print(network.params.disease_params)
print(disease_home)
assert network.params.disease_params == disease_home
print(network.subnets[1].params.disease_params)
print(disease_home)
assert network.subnets[1].params.disease_params == disease_home
print(network.subnets[0].params.disease_params)
print(disease_super)
assert network.subnets[0].params.disease_params == disease_super
infections = network.initialise_infections()
assert infections.N_INF_CLASSES == disease_home.N_INF_CLASSES()
assert \
infections.subinfs[1].N_INF_CLASSES == disease_home.N_INF_CLASSES()
assert \
infections.subinfs[0].N_INF_CLASSES == disease_super.N_INF_CLASSES()
assert disease_super.N_INF_CLASSES() != disease_home.N_INF_CLASSES()
outdir = os.path.join(script_dir, "test_pathway")
with OutputFiles(outdir, force_empty=True, prompt=None) as output_dir:
results = network.copy().run(population=Population(),
output_dir=output_dir,
mixer=mix_evenly,
nthreads=1,
seed=36538943)
# using one thread, but if use 2 then have a system crash after
# any other test that uses the big network. This is because we
# have intialised some global data that assumes a large network,
# which then fails for the small network
OutputFiles.remove(outdir, prompt=None)
print(results[-1])
print(results[-1].initial)
expected = Population(susceptibles=68,
latent=0,
total=0,
recovereds=932,
n_inf_wards=0,
day=72)
print(expected)
assert results[-1].has_equal_SEIR(expected)
assert results[-1].day == expected.day
with OutputFiles(outdir, force_empty=True, prompt=None) as output_dir:
results = network.copy().run(population=Population(),
output_dir=output_dir,
mixer=mix_evenly,
nthreads=1,
seed=36538943)
OutputFiles.remove(outdir, prompt=None)
print(results[-1])
print(results[-1].initial)
print(expected)
assert results[-1].has_equal_SEIR(expected)
assert results[-1].day == expected.day
variables = VariableSet()
print("\nUpdate with null variables")
oldparams = network.params
params = network.params.set_variables(variables)
network.update(params)
assert oldparams == network.params
print(network.params.disease_params)
print(disease_home)
assert network.params.disease_params == disease_home
print(network.subnets[1].params.disease_params)
print(disease_home)
assert network.subnets[1].params.disease_params == disease_home
print(network.subnets[0].params.disease_params)
print(disease_super)
assert network.subnets[0].params.disease_params == disease_super
infections = network.initialise_infections()
assert infections.N_INF_CLASSES == disease_home.N_INF_CLASSES()
assert \
infections.subinfs[1].N_INF_CLASSES == disease_home.N_INF_CLASSES()
assert \
infections.subinfs[0].N_INF_CLASSES == disease_super.N_INF_CLASSES()
assert disease_super.N_INF_CLASSES() != disease_home.N_INF_CLASSES()
outdir = os.path.join(script_dir, "test_pathway")
with OutputFiles(outdir, force_empty=True, prompt=None) as output_dir:
results = network.copy().run(population=Population(),
output_dir=output_dir,
mixer=mix_evenly,
nthreads=1,
seed=36538943)
OutputFiles.remove(outdir, prompt=None)
print(results[-1])
print(expected)
assert results[-1].has_equal_SEIR(expected)
assert results[-1].day == expected.day
def test_go_to(prompt=None, nthreads=1):
seed = 797747
# load all of the parameters
try:
params = Parameters.load(parameters="march29")
except Exception as e:
print(f"Unable to load parameter files. Make sure that you have "
f"cloned the MetaWardsData repository and have set the "
f"environment variable METAWARDSDATA to point to the "
f"local directory containing the repository, e.g. the "
f"default is $HOME/GitHub/MetaWardsData")
raise e
# load the disease and starting-point input files
params.set_disease(os.path.join(script_dir, "data", "ncov.json"))
params.set_input_files("2011Data")
params.add_seeds("ExtraSeedsBrighton.dat")
# the size of the starting population
population = Population(initial=57104043)
nsteps = 20
demographics = Demographics.load(redblue_json)
print("Building the network...")
network = Network.build(params=params)
network = network.specialise(demographics, nthreads=nthreads)
outdir = os.path.join(script_dir, "test_go_to")
with OutputFiles(outdir, force_empty=True, prompt=prompt) as output_dir:
trajectory = network.copy().run(population=population, seed=seed,
output_dir=output_dir,
nsteps=nsteps,
mixer=mix_evenly,
mover=move_red_to_blue,
nthreads=nthreads)
OutputFiles.remove(outdir, prompt=None)
# red demographic was seeded, but all moved to blue, so should have
# no outbreak
pop = trajectory[-1]
print(pop)
assert pop.susceptibles == pop.population
assert pop.subpops[0].population == 0
assert pop.subpops[1].population == pop.population
with OutputFiles(outdir, force_empty=True, prompt=prompt) as output_dir:
trajectory = network.copy().run(population=population, seed=seed,
output_dir=output_dir,
nsteps=nsteps,
mixer=mix_evenly,
mover=move_blue_to_red,
nthreads=nthreads)
OutputFiles.remove(outdir, prompt=None)
# red demographic was seeded so should all be affected
final_pop = trajectory[-1]
print(final_pop)
assert final_pop.susceptibles != final_pop.population
assert final_pop.subpops[0].population == final_pop.population
assert final_pop.subpops[1].population == 0
with OutputFiles(outdir, force_empty=True, prompt=prompt) as output_dir:
trajectory = network.copy().run(population=population, seed=seed,
output_dir=output_dir,
nsteps=nsteps,
mixer=mix_evenly,
mover=move_red_to_blue_2,
nthreads=nthreads)
OutputFiles.remove(outdir, prompt=None)
# red seeded, but all moved to blue. Should have the same outbreak
pop = trajectory[-1]
print(final_pop)
print(pop)
assert pop.susceptibles == final_pop.susceptibles
assert pop.total == final_pop.total
assert pop.latent == final_pop.latent
assert pop.recovereds == final_pop.recovereds
with OutputFiles(outdir, force_empty=True, prompt=prompt) as output_dir:
trajectory = network.copy().run(population=population, seed=seed,
output_dir=output_dir,
nsteps=nsteps,
mixer=mix_evenly,
mover=move_even_odd,
nthreads=nthreads)
OutputFiles.remove(outdir, prompt=None)
# red seeded, but moved back and forth - should all be the same
pop = trajectory[-1]
print(final_pop)
print(pop)
assert pop.susceptibles == final_pop.susceptibles
assert pop.total == final_pop.total
assert pop.latent == final_pop.latent
assert pop.recovereds == final_pop.recovereds
def output_db(population: Population, network: Networks, workspace: Workspace,
output_dir: OutputFiles, **kwargs):
Console.print(f"Calling output_db for a {network.__class__} object")
conn3 = output_dir.open_db("stages.db", initialise=create_tables(network))
c3 = conn3.cursor()
## setup marker for previous daya
for i, subnet in enumerate(network.subnets):
## if first day, then create a copy of the ward data
## these should all be zero at day 0 and so not affect incidence
if not hasattr(workspace.subspaces[i], "output_previous"):
workspace.subspaces[i].output_previous = deepcopy(
workspace.subspaces[i].ward_inf_tot)
for j, sub in enumerate(workspace.subspaces[i].output_previous):
for k, sub1 in enumerate(
workspace.subspaces[i].output_previous[j]):
workspace.subspaces[i].output_previous[j][k] = 0
## extract Rprime and Dprime in each demographic
Rprime = [[] for _ in range(4)]
Dprime = [[] for _ in range(4)]
for i, subnet in enumerate(network.subnets):
## get yesterday's data
ward_inf_previous = workspace.subspaces[i].output_previous
## get today's data
ward_inf_tot = workspace.subspaces[i].ward_inf_tot
## new deaths across wards
if subnet.name != "asymp":
for old, new in zip(ward_inf_previous[3], ward_inf_tot[3]):
Dprime[i].append(new - old)
## new removals across wards
for old, new in zip(ward_inf_previous[2], ward_inf_tot[2]):
Rprime[i].append(new - old)
## calculate Iprime in each demographic
Iprime = [[] for _ in range(4)]
## NEED TO DO FOLLOWING CALCULATIONS IN ORDER
## extract subnets names
sub_names = [network.subnets[i].name for i in range(4)]
## ASYMPTOMATICS
ia = sub_names.index("asymp")
## get data
ward_inf_previous = workspace.subspaces[ia].output_previous
ward_inf_tot = workspace.subspaces[ia].ward_inf_tot
## calculate incidence
for old, new, Rinc in zip(ward_inf_previous[1], ward_inf_tot[1],
Rprime[ia]):
Iprime[ia].append(new - old + Rinc)
## CRITICAL
ic = sub_names.index("critical")
## get data
ward_inf_previous = workspace.subspaces[ic].output_previous
ward_inf_tot = workspace.subspaces[ic].ward_inf_tot
## calculate incidence
for old, new, Rinc, Dinc in zip(ward_inf_previous[1], ward_inf_tot[1],
Rprime[ic], Dprime[ic]):
Iprime[ic].append(new - old + Rinc + Dinc)
## HOSPITAL
ih = sub_names.index("hospital")
## get data
ward_inf_previous = workspace.subspaces[ih].output_previous
ward_inf_tot = workspace.subspaces[ih].ward_inf_tot
## calculate incidence
for old, new, Rinc, Dinc, Cinc in zip(ward_inf_previous[1],
ward_inf_tot[1], Rprime[ih],
Dprime[ih], Iprime[ic]):
Iprime[ih].append(new - old + Rinc + Dinc + Cinc)
## GENPOP
ig = sub_names.index("genpop")
## get data
ward_inf_previous = workspace.subspaces[ig].output_previous
ward_inf_tot = workspace.subspaces[ig].ward_inf_tot
## calculate incidence
for old, new, Rinc, Dinc, Hinc in zip(ward_inf_previous[1],
ward_inf_tot[1], Rprime[ig],
Dprime[ig], Iprime[ih]):
Iprime[ig].append(new - old + Rinc + Dinc + Hinc)
## calculate Eprime in GENPOP demographic
Eprime = []
for old, new, Iinc, Ainc in zip(ward_inf_previous[0], ward_inf_tot[0],
Iprime[ig], Iprime[ia]):
Eprime.append(new - old + Iinc + Ainc)
## loop over wards and write to file
wards = range(0, workspace.subspaces[0].nnodes + 1)
day = [population.day] * len(wards)
# print(workspace.subspaces[0].nnodes)
# print(len(day))
# print(len(wards))
# print(len(Eprime))
## set column names
col_names = ["day", "ward", "Einc", "E", "Iinc", "I", "RI", "DI", "Ainc", "A", "RA", "Hinc", "H",\
"RH", "DH", "Cinc", "C", "RC", "DC"]
col_str = ','.join(col_names)
## extract demographics
asymp_ward = workspace.subspaces[ia].ward_inf_tot
genpop_ward = workspace.subspaces[ig].ward_inf_tot
hospital_ward = workspace.subspaces[ih].ward_inf_tot
critical_ward = workspace.subspaces[ic].ward_inf_tot
## write to file
for day, ward, Einc, E, Iinc, I, RI, DI, Ainc, A, RA, Hinc, H, RH, RD, Cinc, C, RC, DC in\
zip(day, wards, Eprime, genpop_ward[0], Iprime[ig], genpop_ward[1], genpop_ward[2], genpop_ward[3],\
Iprime[ia], asymp_ward[1], asymp_ward[2], Iprime[ih], hospital_ward[1], hospital_ward[2],\
hospital_ward[3], Iprime[ic], critical_ward[1], critical_ward[2], critical_ward[3]):
if ward not in _zero_crossings:
_zero_crossings[ward] = False
## try to fudge a marker for first infections
if Einc != 0 and _zero_crossings[ward] is False and ward != 0:
_zero_crossings[ward] = True
Console.print(f"Got first infection in ward {ward}")
val = [
day, ward, Einc, E, Iinc, I, RI, DI, Ainc, A, RA, Hinc, H, RH, RD,
Cinc, C, RC, DC
]
keeps_str = ",".join([str(v) for v in val])
qstring = f"insert into compact ({col_str}) values ({keeps_str}) "
if _zero_crossings[ward] is True:
c3.execute(qstring)
conn3.commit()
## save today's data so that it can be used tomorrow
for i, subnet in enumerate(network.subnets):
workspace.subspaces[i].output_previous = deepcopy(
workspace.subspaces[i].ward_inf_tot)