def test_make_contacts_with_facilities_from_microstructure(
location='seattle_metro',
state_location='Washington',
country_location='usa',
n=1000):
# First generate microstructure with facilities
sp.make_population(n,
datadir,
location,
state_location,
country_location,
write=False,
plot=False,
return_popdict=True)
# sp.generate_microstructure_with_facilities(datadir, location, state_location,
# country_location, n, sheet_name='United States of America',
# use_two_group_reduction=False, average_LTCF_degree=20, ltcf_staff_age_min=20, ltcf_staff_age_max=60,
# with_school_types=False, school_mixing_type='random', average_class_size=20, inter_grade_mixing=0.1,
# average_student_teacher_ratio=20, average_teacher_teacher_degree=3, teacher_age_min=25, teacher_age_max=75,
# average_student_all_staff_ratio=15, average_additional_staff_degree=20, staff_age_min=20, staff_age_max=75,
# verbose=False, plot=False, write=False, return_popdict=False, use_default=False):
# Then make contacts
popdict = sp.make_population(n, datadir, location, state_location,
country_location)
contains_age = 74
contains_sex = 1
uids = popdict.keys()
assert contains_age in uids
assert contains_sex in uids
return popdict
def test_make_population_with_multi_flags():
with pytest.raises(ValueError,
match=r"Requesting both long term*") as info:
sp.make_population(n=n,
generate=True,
with_industry_code=True,
with_facilities=True)
return info
def test_inter_grade_mixing(school_mixing_type='random'):
"""
When school_mixing_type is 'age_clustered' then inter_grade_mixing should
rewire a fraction of the edges between students in the same age or grade to
be edges with any other student in the school.
When school_mixing_type is 'random' or 'age_and_class_clustered',
inter_grade_mixing has no effect.
Args:
school_mixing_type (str): The mixing type for schools, 'random', 'age_clustered', or 'age_and_class_clustered'.
"""
sp.logger.info(
f'Testing the effect of the parameter inter_grade_mixing for school_mixing_type: {school_mixing_type}'
)
test_pars = sc.dcp(pars)
test_pars['school_mixing_type'] = school_mixing_type
pop_1 = sp.make_population(**test_pars)
test_pars['inter_grade_mixing'] = 0.3
pop_2 = sp.make_population(**test_pars)
# make an adjacency matrix of edges between students
adjm_1 = np.zeros((101, 101))
adjm_2 = np.zeros((101, 101))
student_ids = set()
for i, person in pop_1.items():
if person['sc_student']:
student_ids.add(i)
for ni, i in enumerate(student_ids):
contacts_1 = pop_1[i]['contacts']['S']
contacts_2 = pop_2[i]['contacts']['S']
student_contacts_1 = list(set(contacts_1).intersection(student_ids))
student_contacts_2 = list(set(contacts_2).intersection(student_ids))
age_i = pop_1[i]['age']
for nj, j in enumerate(student_contacts_1):
age_j = pop_1[j]['age']
adjm_1[age_i][age_j] += 1
for nj, j in enumerate(student_contacts_2):
age_j = pop_2[j]['age']
adjm_2[age_i][age_j] += 1
if school_mixing_type in ['random', 'age_and_class_clustered']:
assert np.not_equal(adjm_1, adjm_2).sum(
) == 0, f"inter_grade_mixing parameter check failed. Different values for this parameter produced different results for school_mixing_type: {school_mixing_type}."
elif school_mixing_type in ['age_clustered']:
assert np.not_equal(adjm_1, adjm_2).sum(
) > 0, f"inter_grade_mixing parameter check failed. It produced the same results for different values of this parameter for school_mixing_type: {school_mixing_type}."
def test_api():
n = 50000
max_contacts = {'S': 20, 'W': 10}
population = sp.make_population(n=n, max_contacts=max_contacts)
with pytest.raises(ValueError):
population = sp.make_population(n=298437) # Not a supported number
return population
def test_sizes():
''' Synthpops should support populations down to 200 people '''
nlist = [200, 201, 1001, 9848]
for n in nlist:
print(f'Working on {n}')
pop = sp.make_population(n=n, generate=True)
assert len(pop) == n
with pytest.raises(NotImplementedError):
sp.make_population(n=199, generate=True) # Too small
return pop
def test_api():
''' Basic SynthPops test '''
sp.logger.info('Testing API and logger')
n = 2000
max_contacts = {'S': 20, 'W': 10}
population = sp.make_population(n=n, max_contacts=max_contacts)
with pytest.raises(ValueError):
population = sp.make_population(
n=298437, generate=False) # Not a supported number
return population
def make_pop():
n = [
10000, 10001
][1] # Use either a pre-generated population, or one that has to be made from scratch
max_contacts = {'S': 20, 'W': 10}
population = sp.make_population(n=n, max_contacts=max_contacts)
return population
def make_synthpop(sim):
''' Make a population using synthpops, including contacts '''
import synthpops as sp # Optional import
population = sp.make_population(n=sim['pop_size'])
uids, ages, sexes, contacts = [], [], [], []
for uid,person in population.items():
uids.append(uid)
ages.append(person['age'])
sexes.append(person['sex'])
# Replace contact UIDs with ints...
uid_mapping = {uid:u for u,uid in enumerate(uids)}
key_mapping = {'H':'h', 'S':'s', 'W':'w', 'C':'c'} # Remap keys from old names to new names
for uid,person in population.items():
uid_contacts = person['contacts']
int_contacts = {}
for key in uid_contacts.keys():
new_key = key_mapping[key]
int_contacts[new_key] = []
for uid in uid_contacts[key]:
int_contacts[new_key].append(uid_mapping[uid])
int_contacts[new_key] = np.array(int_contacts[new_key], dtype=np.int32)
contacts.append(int_contacts)
popdict = {}
popdict['uid'] = uids
popdict['age'] = np.array(ages)
popdict['sex'] = np.array(sexes)
popdict['contacts'] = contacts
popdict['layer_keys'] = list(key_mapping.values())
return popdict
def test_make_popdict_generic(n=default_n):
sc.heading(f'Making popdict for {n} people')
n = int(n)
popdict = sp.make_population(
n=n, use_demography=False) # Non-USA not implemented
return popdict
def make_people(sim, verbose=None, id_len=None, die=True):
# Set defaults
if verbose is None: verbose = sim['verbose']
if id_len is None: id_len = 6
# Set inputs
n_people = int(sim['n']) # Shorten
usepopdata = sim['usepopdata'] # Shorten
use_rand_pop = (
usepopdata == 'random'
) # Whether or not to use a random population (as opposed to synthpops)
# Check which type of population to rpoduce
if not use_rand_pop and not cvreqs.available['synthpops']:
errormsg = f'You have requested "{usepopdata}" population, but synthpops is not available; please use "random"'
if die:
raise ValueError(errormsg)
else:
print(errormsg)
usepopdata = 'random'
# Actually create the population
if use_rand_pop:
popdict = make_randpop(sim)
else:
import synthpops as sp # Optional import
popdict = sp.make_population(n=sim['n'])
# Set prognoses by modifying popdict in place
set_prognoses(sim, popdict)
# Actually create the people
people = {
} # Dictionary for storing the people -- use plain dict since faster than odict
for p in range(n_people): # Loop over each person
keys = [
'uid', 'age', 'sex', 'contacts', 'symp_prob', 'severe_prob',
'crit_prob', 'death_prob'
]
person_args = {}
for key in keys:
person_args[key] = popdict[key][p] # Convert from list to dict
person = Person(pars=sim.pars, **person_args) # Create the person
people[person_args['uid']] = person # Save them to the dictionary
# Store UIDs and people
sim.uids = popdict['uid']
sim.people = people
sim.contact_keys = list(sim['contacts_pop'].keys())
average_age = sum(popdict['age'] / n_people)
sc.printv(
f'Created {n_people} people, average age {average_age:0.2f} years', 1,
verbose)
return
def test_Spokane():
"""Test that a Dakar population can be created with the basic SynthPops API."""
sp.logger.info("Test that a Spokane population can be created with the basic SynthPops API.")
pop = sp.make_population(**pars)
age_distr = sp.read_age_bracket_distr(sp.datadir, country_location='usa', state_location='Washington', location='seattle_metro')
assert len(age_distr) == 20, f'Check failed, len(age_distr): {len(age_distr)}' # will remove if this passes in github actions test
sp.set_location_defaults('defaults') # Reset default values after this test is complete.
return pop
def make_pop(n):
"""
run function to create n population for input n
"""
# n = [10000, 10001][1] # Use either a pre-generated population, or one that has to be made from scratch
max_contacts = {
'W': 10
} # only workplaces get trimmed. Schools have a separate method in the schools_module
population = sp.make_population(n=n, max_contacts=max_contacts)
return population
def test_api():
''' Basic SynthPops test '''
sp.logger.info('Testing API and logger')
n = 2000
max_contacts = {'S': 20, 'W': 10}
population = sp.make_population(n=n, max_contacts=max_contacts)
return population
def test_sizes():
''' Synthpops should support populations down to 200 people '''
nlist = [100, 246]
for n in nlist:
print(f'Working on {n}')
pop = sp.make_population(n=n, generate=True)
assert len(pop) == n
return pop
def test_generate_microstructures_with_facilities():
popdict = sp.make_population(datadir=datadir,
location=location,
state_location=state_location,
country_location=country_location,
n=n,
write=write,
plot=False,
return_popdict=return_popdict)
assert (len(popdict) is not None)
return popdict
def test_without_school_types():
"""
Test that without school types, all schools are put together in one group.
"""
sp.logger.info("Creating schools where with_school_types is False.")
test_pars = sc.dcp(pars)
test_pars['with_school_types'] = None
pop = sp.make_population(**test_pars)
fig, ax, school_types = plot_school_sizes_by_type(pop,
test_pars,
do_show=True)
return pop, fig, ax, school_types
def test_plot_generated_trimmed_contact_matrix(setting_code='H', n=5000, aggregate_flag=True, logcolors_flag=True,
density_or_frequency='density', with_facilities=False, cmap='cmr.freeze_r', fontsize=16, rotation=50):
"""
Plot the age mixing matrix for a specific setting where the edges are trimmed.
Args:
setting_code (str) : name of the physial contact setting: H for households, S for schools, W for workplaces, C for community or other
n (int) : number of people in the population
aggregate_flag (book) : If True, plot the contact matrix for aggregate age brackets, else single year age contact matrix.
logcolors_flag (bool) : If True, plot heatmap in logscale
density_or_frequency (str) : If 'density', then each contact counts for 1/(group size -1) of a person's contact in a group, elif 'frequency' then count each contact. This means that more people in a group leads to higher rates of contact/exposure.
with_facilities (bool) : If True, create long term care facilities
cmap(str or matplotlib colormap) : colormap
fontsize (int) : base font size
rotation (int) : rotation for x axis labels
Returns:
A fig object.
"""
datadir = sp.datadir
state_location = 'Washington'
location = 'seattle_metro'
country_location = 'usa'
# popdict = {}
options_args = {'use_microstructure': True}
network_distr_args = {'Npop': int(n)}
# contacts = sp.make_contacts(popdict, state_location=state_location, location=location, options_args=options_args,
# network_distr_args=network_distr_args)
# contacts = sp.trim_contacts(contacts, trimmed_size_dic=None, use_clusters=False)
population = sp.make_population(n, generate=True, with_facilities=with_facilities)
age_brackets = sp.get_census_age_brackets(datadir, state_location=state_location, country_location=country_location)
age_by_brackets_dic = sp.get_age_by_brackets_dic(age_brackets)
ages = []
for uid in population:
ages.append(population[uid]['age'])
age_count = Counter(ages)
aggregate_age_count = sp.get_aggregate_ages(age_count, age_by_brackets_dic)
matrix = sp.calculate_contact_matrix(population, density_or_frequency, setting_code)
fig = sp.plot_contact_matrix(matrix, age_count, aggregate_age_count, age_brackets, age_by_brackets_dic,
setting_code, density_or_frequency, logcolors_flag, aggregate_flag, cmap, fontsize, rotation)
return fig
def make_synthpop(sim, directed=False):
''' Make a population using synthpops, including contacts '''
import synthpops as sp # Optional import
pop_size = sim['pop_size']
population = sp.make_population(n=pop_size, generate=True)
uids, ages, sexes, contacts = [], [], [], []
for uid, person in population.items():
uids.append(uid)
ages.append(person['age'])
sexes.append(person['sex'])
# Replace contact UIDs with ints
uid_mapping = {uid: u for u, uid in enumerate(uids)}
key_mapping = {
'H': 'h',
'S': 's',
'W': 'w',
'C': 'c'
} # Remap keys from old names to new names
for uid in uids:
iid = uid_mapping[uid] # Integer UID
person = population.pop(uid)
uid_contacts = sc.dcp(person['contacts'])
int_contacts = {}
for spkey in uid_contacts.keys():
lkey = key_mapping[
spkey] # Map the SynthPops key into a Covasim layer key
int_contacts[lkey] = []
for cid in uid_contacts[spkey]: # Contact ID
icid = uid_mapping[cid] # Integer contact ID
if icid > iid or directed: # Don't add duplicate contacts
int_contacts[lkey].append(icid)
int_contacts[lkey] = np.array(int_contacts[lkey],
dtype=cvd.default_int)
contacts.append(int_contacts)
# Add community contacts
c_contacts, _ = make_random_contacts(pop_size, {'c': sim['contacts']['c']})
for i in range(pop_size):
contacts[i]['c'] = c_contacts[i][
'c'] # Copy over community contacts -- present for everyone
# Finalize
popdict = {}
popdict['uid'] = sc.dcp(uids)
popdict['age'] = np.array(ages)
popdict['sex'] = np.array(sexes)
popdict['contacts'] = sc.dcp(contacts)
layer_keys = list(key_mapping.values())
return popdict, layer_keys
def test_with_non_teaching_staff():
"""
When with_non_teaching_staff is True, each school should be created with non
teaching staff. Otherwise when False, there should be no non teaching staff
in schools.
"""
sp.logger.info(
f'Testing the effect of the parameter with_non_teaching_staff.')
test_pars = sc.dcp(pars)
test_pars['with_non_teaching_staff'] = True
pop_1 = sp.make_population(**test_pars)
test_pars['with_non_teaching_staff'] = False
pop_2 = sp.make_population(**test_pars)
school_staff_1 = {}
for i, person in pop_1.items():
if person['scid'] is not None:
school_staff_1.setdefault(person['scid'], 0)
if person['sc_staff']:
school_staff_1[person['scid']] += 1
staff_ids_2 = set()
for i, person in pop_2.items():
if person['sc_staff']:
staff_ids_2.add(i)
# with_non_teaching_staff == True so minimum number of staff per school needs to be 1
assert min(
school_staff_1.values()
) >= 1, f"with_non_teaching_staff parameter check failed when set to True."
# with_non_teaching_staff == False so there should be no one who shows up with sc_staff set to 1
assert len(
staff_ids_2
) == 0, f"with_non_teaching_staff parameter check failed when set to False."
def test_original_household_method(do_show=False):
sp.logger.info("Generating households with the infer_ages method.")
test_pars = sc.dcp(pars)
test_pars['household_method'] = 'infer_ages'
pop = sp.make_population(**test_pars)
datadir = sp.datadir
fig, ax = plot_age_dist(datadir, pop, test_pars, do_show,
test_pars['household_method'])
if do_show:
plt.show()
return pop
def test_Dakar():
"""Test that a Dakar population can be created with the basic SynthPops API."""
sp.logger.info("Test that a Dakar population can be created with the basic SynthPops API.")
sp.set_nbrackets(18) # Dakar age distributions available are up to 18 age brackets
pop = sp.make_population(**pars)
assert len(pop) == pars['n'], 'Check failed.'
print('Check passed')
sp.set_location_defaults('defaults') # Reset default values after this test is complete.
sp.logger.info("Test that the default country was reset.")
assert sp.default_country == 'usa', f'Check failed: default_country is {sp.default_country}'
print('2nd Check passed')
return pop
def test_make_population_with_industry_code():
popdict = sp.make_population(n=n, generate=True, with_industry_code=True, use_two_group_reduction=True,
average_LTCF_degree=20)
# verify these keys are either None or set to a value
for i, uid in enumerate(popdict):
if popdict[uid]['wpid'] is None:
assert popdict[uid]['wpid'] is None
else:
assert popdict[uid]['wpid'] is not None
if popdict[uid]['wpindcode'] is None:
assert popdict[uid]['wpindcode'] is None
else:
assert popdict[uid]['wpindcode'] is not None
return popdict
def test_basic_api():
''' Basic SynthPops test '''
sp.logger.info('Testing basic API')
pop = sp.make_population(**pars)
if regenerate or not os.path.exists(outfile):
print('Saving...')
sc.saveobj(outfile, pop)
else:
print('Checking...')
pop2 = sc.loadobj(outfile)
assert pop == pop2, 'Check failed'
print('Check passed')
return pop
def make_synthpop(sim):
''' Make a population using synthpops, including contacts '''
import synthpops as sp # Optional import
pop_size = sim['pop_size']
population = sp.make_population(n=pop_size)
uids, ages, sexes, contacts = [], [], [], []
for uid, person in population.items():
uids.append(uid)
ages.append(person['age'])
sexes.append(person['sex'])
# Replace contact UIDs with ints...
uid_mapping = {uid: u for u, uid in enumerate(uids)}
key_mapping = {
'H': 'h',
'S': 's',
'W': 'w',
'C': 'c'
} # Remap keys from old names to new names
for uid in uids:
person = population.pop(uid)
uid_contacts = sc.dcp(person['contacts'])
int_contacts = {}
for key in uid_contacts.keys():
new_key = key_mapping[key]
int_contacts[new_key] = []
for uid in uid_contacts[key]:
int_contacts[new_key].append(uid_mapping[uid])
int_contacts[new_key] = np.array(int_contacts[new_key],
dtype=cvd.default_int)
contacts.append(int_contacts)
# Add community contacts
c_contacts, _ = make_random_contacts(pop_size, {'c': sim['contacts']['c']})
for i in range(pop_size):
contacts[i]['c'] = c_contacts[i][
'c'] # Copy over community contacts -- present for everyone
# Finalize
popdict = {}
popdict['uid'] = sc.dcp(uids)
popdict['age'] = np.array(ages)
popdict['sex'] = np.array(sexes)
popdict['contacts'] = sc.dcp(contacts)
layer_keys = list(key_mapping.values())
return popdict, layer_keys
def runpop(self, filename, actual_vals, testprefix = "test"):
# if default sort order is not concerned:
# pop = dict(sorted(pop.items(), key=lambda x: x[0]))
params = {}
args = inspect.getfullargspec(sp.make_population).args
defaults = inspect.getfullargspec(sp.make_population).defaults
for i in range(0, len(args)):
params[args[i]] = defaults[i]
for name in actual_vals:
if name in params.keys():
params[name] = actual_vals[name]
with open(os.path.join(self.configDir, f"{testprefix}.txt"), mode="w") as cf:
for key, value in params.items():
cf.writelines(str(key) + ':' + str(value) + "\n")
pop = sp.make_population(**params)
return pop
def test_school_types_created():
"""
Test that unique school types are created.
Returns:
A list of the school types expected for the specified location.
"""
sp.logger.info(
f"Test that unique school types are created for each school.\nRun this first to see what school types you are working with."
)
test_pars = sc.dcp(pars)
test_pars['n'] = 20e3
pop = sp.make_population(**test_pars)
if pars['with_school_types']:
expected_school_size_distr = sp.get_school_size_distr_by_type(
sp.datadir,
location=pars['location'],
state_location=pars['state_location'],
country_location=pars['country_location'],
use_default=pars['use_default'])
expected_school_types = sorted(expected_school_size_distr.keys())
else:
expected_school_types = [None]
schools_by_type = dict()
for i, person in pop.items():
if person['scid'] is not None:
schools_by_type.setdefault(person['scid'], set())
schools_by_type[person['scid']].add(person['sc_type'])
for s, school_type in schools_by_type.items():
assert len(
school_type
) == 1, f'Check failed. School {s} is listed as having more than one type.'
schools_by_type[s] = list(school_type)[0]
gen_school_types = sorted(set(schools_by_type.values()))
assert gen_school_types == expected_school_types, f"Check failed. generated types: {gen_school_types}, expected: {expected_school_types}"
print(
f"School types generated for {test_pars['location']}: {set(schools_by_type.values())}"
)
return list(set(schools_by_type.values()))
def test_make_population():
population = sp.make_population(n=n, generate=True, with_facilities=True, use_two_group_reduction=True,
average_LTCF_degree=20)
for key, person in population.items():
assert population[key]['contacts'] is not None
assert population[key]['contacts']['LTCF'] is not None
assert len(population[key]['contacts']['H']) >= 0
expected_layers = {'H', 'S', 'W', 'C', 'LTCF'}
for layerkey in population[key]['contacts'].keys():
if layerkey in expected_layers:
assert True
else:
assert False
return population
def test_school_sizes_by_type(pars, do_show=False):
"""
Visually show how the school size distribution generated compares to the
data for the location being simulated.
Notes:
The larger the population size, the better the generated school size
distributions by school type can match the expected data. If generated
populations are too small, larger schools will be missed and in
general there won't be enough schools generated to apply statistical
tests.
"""
sp.logger.info(
"Creating schools by school type and a visual comparison of how they match to data."
)
pop = sp.make_population(**pars)
fig, ax, school_types = plot_school_sizes_by_type(pop, pars, do_show=True)
return pop, fig, ax, school_types
def test_smoothed_and_fixed_ages_household_method(do_show=False):
sp.logger.info(
"Generating households with the fixed_ages and smoothed_ages methods.")
test_pars = sc.dcp(pars)
test_pars['location'] = 'Spokane_County'
test_pars['household_method'] = 'fixed_ages'
test_pars['smooth_ages'] = True
test_pars['window_length'] = 7 # window for averaging the age distribution
pop = sp.make_population(**test_pars)
datadir = sp.datadir
fig, ax = plot_age_dist(datadir, pop, test_pars, do_show,
test_pars['household_method'])
if do_show:
plt.show()
return fig, ax
def test_make_population(location='seattle_metro',
state_location='Washington',
n=10000):
contacts = sp.make_population(datadir=datadir,
location=location,
state_location=state_location,
country_location=country_location,
n=n,
with_industry_code=False)
uids = contacts.keys()
uids = [uid for uid in uids]
for n, uid in enumerate(uids):
if n > 20:
break
layers = contacts[uid]['contacts']
print('uid', uid, 'age', contacts[uid]['age'], 'total contacts',
np.sum([len(contacts[uid]['contacts'][k]) for k in layers]))
for k in layers:
contact_ages = [
contacts[c]['age'] for c in contacts[uid]['contacts'][k]
]
print(k, len(contact_ages), 'contact ages', contact_ages)
schools = contacts[uid]['contacts']['S']
school_id = contacts[uid]['scid']
school_teacher = contacts[uid]['sc_teacher']
school_student = contacts[uid]['sc_student']
assert schools is not None
if school_id is None:
assert school_id is None
else:
assert school_id is not None
if school_teacher is None:
assert school_teacher is None
else:
assert school_teacher == 1
if school_student is None:
assert school_student is None
else:
assert school_student == 1
return contacts
