def check_similarity(self, actual, expected):
"""
Compare two population dictionaries using contact matrix
Assuming the canberra distance should be close to zero
"""
for code in ['H', 'W', 'S']:
for option in ['density', 'frequency']:
print(f"\ncheck:{code} with {option}")
actual_matrix = sp.calculate_contact_matrix(actual, density_or_frequency=option, setting_code=code)
expected_matrix = sp.calculate_contact_matrix(expected, density_or_frequency=option, setting_code=code)
# calculate Canberra distance
# assuming they should round to 0
d = distance.canberra(actual_matrix.flatten(), expected_matrix.flatten())
self.assertEqual(round(d), 0, f"actual distance for {code}/{option} is {str(round(d))}, "
f"you need to uncommented line 55 to plot the density matrix and investigate!")
def test_plot_generated_trimmed_contact_matrix(setting_code='H', n=5000, aggregate_flag=True, logcolors_flag=True,
density_or_frequency='density'):
datadir = sp.datadir
state_location = 'Washington'
location = 'seattle_metro'
country_location = 'usa'
popdict = {}
options_args = {'use_microstructure': True}
network_distr_args = {'Npop': int(n)}
contacts = sp.make_contacts(popdict, state_location=state_location, location=location, options_args=options_args,
network_distr_args=network_distr_args)
contacts = sp.trim_contacts(contacts, trimmed_size_dic=None, use_clusters=False)
age_brackets = sp.get_census_age_brackets(datadir, state_location=state_location, country_location=country_location)
age_by_brackets_dic = sp.get_age_by_brackets_dic(age_brackets)
ages = []
for uid in contacts:
ages.append(contacts[uid]['age'])
age_count = Counter(ages)
aggregate_age_count = sp.get_aggregate_ages(age_count, age_by_brackets_dic)
freq_matrix_dic = sp.calculate_contact_matrix(contacts, density_or_frequency)
fig = sp.plot_contact_frequency(freq_matrix_dic, age_count, aggregate_age_count, age_brackets, age_by_brackets_dic,
setting_code, density_or_frequency, logcolors_flag, aggregate_flag)
return fig
def get_pop_details(self,
pop,
dir,
title_prefix,
location,
state_location,
country_location,
decimal=3):
os.makedirs(dir, exist_ok=True)
# want default age brackets and to see that they line up with the average contacts by age bracket created
age_brackets = spdd.get_census_age_brackets(self.datadir, location,
state_location,
country_location)
age_brackets_labels = [
str(age_brackets[b][0]) + '-' + str(age_brackets[b][-1])
for b in sorted(age_brackets.keys())
]
for setting_code in ['H', 'W', 'S']:
average_contacts = utilities.get_average_contact_by_age(
pop, self.datadir, setting_code=setting_code, decimal=decimal)
fmt = f'%.{str(decimal)}f'
# print(f"expected contacts by age for {code}:\n", average_contacts)
utilities.plot_array(
average_contacts,
datadir=self.figDir,
testprefix=
f"{self.n}_seed_{self.seed}_{setting_code}_average_contacts",
expect_label='Expected' if self.generateBaseline else 'Test',
names=age_brackets_labels,
xlabel_rotation=50)
sc.savejson(os.path.join(
dir,
f"{self.n}_seed_{self.seed}_{setting_code}_average_contact.json"
),
dict(enumerate(average_contacts.tolist())),
indent=2)
for method in ['density', 'frequency']:
matrix = sp.calculate_contact_matrix(pop, method, setting_code)
brackets = spdd.get_census_age_brackets(
self.datadir, location, state_location, country_location)
ageindex = spb.get_age_by_brackets_dic(brackets)
agg_matrix = spb.get_aggregate_matrix(matrix, ageindex)
textfile = os.path.join(
dir,
f"{self.n}_seed_{self.seed}_{setting_code}_{method}_contact_matrix.csv"
)
np.savetxt(textfile, agg_matrix, delimiter=",", fmt=fmt)
fig = sp.plot_contacts(pop,
setting_code=setting_code,
density_or_frequency=method,
do_show=False)
fig.savefig(
os.path.join(
self.figDir,
f"{self.n}_seed_{self.seed}_{setting_code}_{method}_contact_matrix.png"
))
def check_similarity(self, actual, expected):
"""
Compare two population dictionaries using contact matrix
Assuming the canberra distance should be small
"""
passed = True
checked = False
for code in ['H', 'W', 'S']:
# check average contacts per age, if difference is small enough to tolerate
expected_contacts = utilities.get_average_contact_by_age(
expected, self.datadir, code=code)
print("expected contacts by age:\n", expected_contacts)
np.savetxt(os.path.join(
self.pdfDir,
f"pop{self.n}_seed_{self.seed}_{code}_average_contact.csv"),
expected_contacts,
delimiter=",")
actual_contacts = utilities.get_average_contact_by_age(
actual, self.datadir, code=code)
print("actual contacts by age:\n", actual_contacts)
max_difference = np.abs(expected_contacts - actual_contacts).max()
checked = True
print(
f"max_difference for contacts {code} in each age bracket:{max_difference}\n"
)
if max_difference > 1:
for option in ['density', 'frequency']:
print(f"\ncheck:{code} with {option}")
actual_matrix = sp.calculate_contact_matrix(
actual, density_or_frequency=option, setting_code=code)
# expected_matrix = sp.calculate_contact_matrix(expected, density_or_frequency=option, setting_code=code)
expected_matrix = np.loadtxt(os.path.join(
self.expectedDir,
f"pop{self.n}_seed_{self.seed}_{code}_{option}_contact_matrix.csv"
),
unpack=True,
delimiter=",")
np.savetxt(os.path.join(
self.pdfDir,
f"pop{self.n}_seed_{self.seed}_{code}_{option}_contact_matrix.csv"
),
expected_matrix,
delimiter=",")
# calculate Canberra distance
# assuming they should round to 0
d = np.abs(expected_matrix - actual_matrix).mean()
# d = distance.canberra(actual_matrix.flatten(), expected_matrix.flatten())
print(
f"mean absolute difference between actual/expected contact matrix for {code}/{option} is {str(round(d, 3))}"
)
if d > 1:
passed = passed & False
return checked & passed
def get_pop_details(self,
pop,
dir,
title_prefix,
location,
state_location,
country_location,
decimal=3):
os.makedirs(dir, exist_ok=True)
for setting_code in ['H', 'W', 'S']:
average_contacts = utilities.get_average_contact_by_age(
pop, self.datadir, setting_code=setting_code, decimal=decimal)
fmt = f'%.{str(decimal)}f'
# print(f"expected contacts by age for {code}:\n", average_contacts)
utilities.plot_array(
average_contacts,
datadir=self.figDir,
testprefix=
f"{self.n}_seed_{self.seed}_{setting_code}_average_contacts",
expect_label='Expected' if self.generateBaseline else 'Test')
sc.savejson(os.path.join(
dir,
f"{self.n}_seed_{self.seed}_{setting_code}_average_contact.json"
),
dict(enumerate(average_contacts.tolist())),
indent=2)
for type in ['density', 'frequency']:
matrix = sp.calculate_contact_matrix(pop, type, setting_code)
brackets = sp.get_census_age_brackets(self.datadir,
state_location,
country_location)
ageindex = sp.get_age_by_brackets_dic(brackets)
agg_matrix = sp.get_aggregate_matrix(matrix, ageindex)
np.savetxt(os.path.join(
dir,
f"{self.n}_seed_{self.seed}_{setting_code}_{type}_contact_matrix.csv"
),
agg_matrix,
delimiter=",",
fmt=fmt)
fig = plot_age_mixing_matrices.test_plot_generated_contact_matrix(
setting_code=setting_code,
population=pop,
title_prefix=" Expected "
if self.generateBaseline else " Test ",
density_or_frequency=type)
# fig.show()
fig.savefig(
os.path.join(
self.figDir,
f"{self.n}_seed_{self.seed}_{setting_code}_{type}_contact_matrix.png"
))
def test_plot_generated_trimmed_contact_matrix(setting_code='H', n=5000, aggregate_flag=True, logcolors_flag=True,
density_or_frequency='density', with_facilities=False, cmap='cmr.freeze_r', fontsize=16, rotation=50):
"""
Plot the age mixing matrix for a specific setting where the edges are trimmed.
Args:
setting_code (str) : name of the physial contact setting: H for households, S for schools, W for workplaces, C for community or other
n (int) : number of people in the population
aggregate_flag (book) : If True, plot the contact matrix for aggregate age brackets, else single year age contact matrix.
logcolors_flag (bool) : If True, plot heatmap in logscale
density_or_frequency (str) : If 'density', then each contact counts for 1/(group size -1) of a person's contact in a group, elif 'frequency' then count each contact. This means that more people in a group leads to higher rates of contact/exposure.
with_facilities (bool) : If True, create long term care facilities
cmap(str or matplotlib colormap) : colormap
fontsize (int) : base font size
rotation (int) : rotation for x axis labels
Returns:
A fig object.
"""
datadir = sp.datadir
state_location = 'Washington'
location = 'seattle_metro'
country_location = 'usa'
# popdict = {}
options_args = {'use_microstructure': True}
network_distr_args = {'Npop': int(n)}
# contacts = sp.make_contacts(popdict, state_location=state_location, location=location, options_args=options_args,
# network_distr_args=network_distr_args)
# contacts = sp.trim_contacts(contacts, trimmed_size_dic=None, use_clusters=False)
population = sp.make_population(n, generate=True, with_facilities=with_facilities)
age_brackets = sp.get_census_age_brackets(datadir, state_location=state_location, country_location=country_location)
age_by_brackets_dic = sp.get_age_by_brackets_dic(age_brackets)
ages = []
for uid in population:
ages.append(population[uid]['age'])
age_count = Counter(ages)
aggregate_age_count = sp.get_aggregate_ages(age_count, age_by_brackets_dic)
matrix = sp.calculate_contact_matrix(population, density_or_frequency, setting_code)
fig = sp.plot_contact_matrix(matrix, age_count, aggregate_age_count, age_brackets, age_by_brackets_dic,
setting_code, density_or_frequency, logcolors_flag, aggregate_flag, cmap, fontsize, rotation)
return fig
def get_pop_contact_matrix(pop, layer, method):
"""
Args:
pop (pop object) : population, either synthpops.pop.Pop, covasim.people.People, or dict
layer (str) : name of the physical contact layer: H for households, S for schools, W for workplaces, C for community or other
method: (str) : density or frequency
Returns:
array: contact matrix
"""
age_brackets = spdd.get_census_age_brackets(**pop.loc_pars)
matrix = sp.calculate_contact_matrix(pop.popdict, method, layer)
ageindex = spb.get_age_by_brackets(age_brackets)
agg_matrix = spb.get_aggregate_matrix(matrix, ageindex)
return agg_matrix
def test_older_ages_have_household_contacts():
"""
Test that older age groups (85+) have at least some household contacts with
other older individuals if expected. Together, if sp.Pop.generate() uses the
incorrect number of age brackets with the contact matrices, older age groups
will not be generated as household contacts for each other (when we look at
the generated contact matrix for households, the blocks between 85+ year
olds would then be 0 for relatively large populations, even though the
household contact matrix would have us expect otherwise.)
"""
test_pars = sc.dcp(pars)
test_pars['n'] = 20e3
pop = sp.Pop(**test_pars)
pop_dict = pop.to_dict()
contact_matrix_dic = sp.get_contact_matrix_dic(sp.datadir,
sheet_name=pop.sheet_name)
contact_matrix_nbrackets = contact_matrix_dic[list(
contact_matrix_dic.keys())[0]].shape[0]
cm_age_brackets = sp.get_census_age_brackets(
sp.datadir,
country_location=pop.country_location,
state_location=pop.state_location,
location=pop.location,
nbrackets=contact_matrix_nbrackets)
cm_age_by_brackets_dic = sp.get_age_by_brackets_dic(cm_age_brackets)
age_threshold = 85
age_threshold_bracket = cm_age_by_brackets_dic[age_threshold]
expected_older_contact = np.sum(
contact_matrix_dic['H'][age_threshold_bracket:,
age_threshold_bracket:])
matrix = sp.calculate_contact_matrix(pop_dict, setting_code='H')
gen_older_age_contacts = np.sum(matrix[age_threshold:, age_threshold:])
if expected_older_contact != 0:
assert gen_older_age_contacts != 0, f'Check failed, individuals over {age_threshold} years old have no contacts with each other in households even though the household contact matrix expects them to.'
else:
assert gen_older_age_contacts == 0, f'Check failed, individuals over {age_threshold} years old have {gen_older_age_contacts} household contacts with each other even though the household contact matrix expects them to have none.'
print('Check passed.')
def test_older_ages_have_household_contacts():
"""
Test that older age groups (80+) have at least some household contacts with
other older individuals if expected. Together, if sp.Pop.generate() uses the
incorrect number of age brackets with the contact matrices, older age groups
will not be generated as household contacts for each other (when we look at
the generated contact matrix for households, the blocks between 85+ year
olds would then be 0 for relatively large populations, even though the
household contact matrix would have us expect otherwise.)
"""
test_pars = sc.dcp(pars)
test_pars.n = settings.pop_sizes.medium_large # decent size to check older populations in households
pop = sp.Pop(**test_pars)
pop_dict = pop.to_dict()
loc_pars = pop.loc_pars
contact_matrices = sp.get_contact_matrices(sp.settings.datadir,
sheet_name=pop.sheet_name)
contact_matrix_nbrackets = contact_matrices[list(
contact_matrices.keys())[0]].shape[0]
cm_age_brackets = sp.get_census_age_brackets(
**sc.mergedicts(loc_pars, {'nbrackets': contact_matrix_nbrackets}))
cm_age_by_brackets = sp.get_age_by_brackets(cm_age_brackets)
age_threshold = 80
age_threshold_bracket = cm_age_by_brackets[age_threshold]
expected_older_contact = np.sum(
contact_matrices['H'][age_threshold_bracket:, age_threshold_bracket:])
matrix = sp.calculate_contact_matrix(pop_dict, layer='H')
gen_older_age_contacts = np.sum(matrix[age_threshold:, age_threshold:])
if expected_older_contact != 0:
assert gen_older_age_contacts != 0, f'Check failed, individuals over {age_threshold} years old have no contacts with each other in households even though the household contact matrix expects them to.'
else:
assert gen_older_age_contacts == 0, f'Check failed, individuals over {age_threshold} years old have {gen_older_age_contacts} household contacts with each other even though the household contact matrix expects them to have none.'
print('Check passed.')
vbounds['S'] = {'vmin': 1e-2, 'vmax': 1e0}
vbounds['W'] = {'vmin': 1e-2, 'vmax': 1e0}
vbounds['LTCF'] = {'vmin': 1e-2, 'vmax': 1e-0}
im = []
cax = []
cbar = []
rotation = 66
for l, layer in enumerate(keys_to_plot):
setting_code = layer.title()
if setting_code == 'L':
setting_code = 'LTCF'
print(f'Plotting average age mixing contact matrix in layer: {layer}')
matrix_dic[layer] = sp.calculate_contact_matrix(population,
density_or_frequency,
setting_code)
if aggregate_flag:
aggregate_matrix = sp.get_aggregate_matrix(matrix_dic[layer],
age_by_brackets_dic)
asymmetric_matrix = sp.get_asymmetric_matrix(aggregate_matrix,
aggregate_age_count)
else:
asymmetric_matrix = sp.get_asymmetric_matrix(matrix_dic[layer],
age_count)
im.append(ax[2 * l].imshow(asymmetric_matrix.T,
origin='lower',
interpolation='nearest',
