def check_similarity(self, actual, expected): """ Compare two population dictionaries using contact matrix Assuming the canberra distance should be close to zero """ for code in ['H', 'W', 'S']: for option in ['density', 'frequency']: print(f"\ncheck:{code} with {option}") actual_matrix = sp.calculate_contact_matrix(actual, density_or_frequency=option, setting_code=code) expected_matrix = sp.calculate_contact_matrix(expected, density_or_frequency=option, setting_code=code) # calculate Canberra distance # assuming they should round to 0 d = distance.canberra(actual_matrix.flatten(), expected_matrix.flatten()) self.assertEqual(round(d), 0, f"actual distance for {code}/{option} is {str(round(d))}, " f"you need to uncommented line 55 to plot the density matrix and investigate!")
def test_plot_generated_trimmed_contact_matrix(setting_code='H', n=5000, aggregate_flag=True, logcolors_flag=True, density_or_frequency='density'): datadir = sp.datadir state_location = 'Washington' location = 'seattle_metro' country_location = 'usa' popdict = {} options_args = {'use_microstructure': True} network_distr_args = {'Npop': int(n)} contacts = sp.make_contacts(popdict, state_location=state_location, location=location, options_args=options_args, network_distr_args=network_distr_args) contacts = sp.trim_contacts(contacts, trimmed_size_dic=None, use_clusters=False) age_brackets = sp.get_census_age_brackets(datadir, state_location=state_location, country_location=country_location) age_by_brackets_dic = sp.get_age_by_brackets_dic(age_brackets) ages = [] for uid in contacts: ages.append(contacts[uid]['age']) age_count = Counter(ages) aggregate_age_count = sp.get_aggregate_ages(age_count, age_by_brackets_dic) freq_matrix_dic = sp.calculate_contact_matrix(contacts, density_or_frequency) fig = sp.plot_contact_frequency(freq_matrix_dic, age_count, aggregate_age_count, age_brackets, age_by_brackets_dic, setting_code, density_or_frequency, logcolors_flag, aggregate_flag) return fig
def get_pop_details(self, pop, dir, title_prefix, location, state_location, country_location, decimal=3): os.makedirs(dir, exist_ok=True) # want default age brackets and to see that they line up with the average contacts by age bracket created age_brackets = spdd.get_census_age_brackets(self.datadir, location, state_location, country_location) age_brackets_labels = [ str(age_brackets[b][0]) + '-' + str(age_brackets[b][-1]) for b in sorted(age_brackets.keys()) ] for setting_code in ['H', 'W', 'S']: average_contacts = utilities.get_average_contact_by_age( pop, self.datadir, setting_code=setting_code, decimal=decimal) fmt = f'%.{str(decimal)}f' # print(f"expected contacts by age for {code}:\n", average_contacts) utilities.plot_array( average_contacts, datadir=self.figDir, testprefix= f"{self.n}_seed_{self.seed}_{setting_code}_average_contacts", expect_label='Expected' if self.generateBaseline else 'Test', names=age_brackets_labels, xlabel_rotation=50) sc.savejson(os.path.join( dir, f"{self.n}_seed_{self.seed}_{setting_code}_average_contact.json" ), dict(enumerate(average_contacts.tolist())), indent=2) for method in ['density', 'frequency']: matrix = sp.calculate_contact_matrix(pop, method, setting_code) brackets = spdd.get_census_age_brackets( self.datadir, location, state_location, country_location) ageindex = spb.get_age_by_brackets_dic(brackets) agg_matrix = spb.get_aggregate_matrix(matrix, ageindex) textfile = os.path.join( dir, f"{self.n}_seed_{self.seed}_{setting_code}_{method}_contact_matrix.csv" ) np.savetxt(textfile, agg_matrix, delimiter=",", fmt=fmt) fig = sp.plot_contacts(pop, setting_code=setting_code, density_or_frequency=method, do_show=False) fig.savefig( os.path.join( self.figDir, f"{self.n}_seed_{self.seed}_{setting_code}_{method}_contact_matrix.png" ))
def check_similarity(self, actual, expected): """ Compare two population dictionaries using contact matrix Assuming the canberra distance should be small """ passed = True checked = False for code in ['H', 'W', 'S']: # check average contacts per age, if difference is small enough to tolerate expected_contacts = utilities.get_average_contact_by_age( expected, self.datadir, code=code) print("expected contacts by age:\n", expected_contacts) np.savetxt(os.path.join( self.pdfDir, f"pop{self.n}_seed_{self.seed}_{code}_average_contact.csv"), expected_contacts, delimiter=",") actual_contacts = utilities.get_average_contact_by_age( actual, self.datadir, code=code) print("actual contacts by age:\n", actual_contacts) max_difference = np.abs(expected_contacts - actual_contacts).max() checked = True print( f"max_difference for contacts {code} in each age bracket:{max_difference}\n" ) if max_difference > 1: for option in ['density', 'frequency']: print(f"\ncheck:{code} with {option}") actual_matrix = sp.calculate_contact_matrix( actual, density_or_frequency=option, setting_code=code) # expected_matrix = sp.calculate_contact_matrix(expected, density_or_frequency=option, setting_code=code) expected_matrix = np.loadtxt(os.path.join( self.expectedDir, f"pop{self.n}_seed_{self.seed}_{code}_{option}_contact_matrix.csv" ), unpack=True, delimiter=",") np.savetxt(os.path.join( self.pdfDir, f"pop{self.n}_seed_{self.seed}_{code}_{option}_contact_matrix.csv" ), expected_matrix, delimiter=",") # calculate Canberra distance # assuming they should round to 0 d = np.abs(expected_matrix - actual_matrix).mean() # d = distance.canberra(actual_matrix.flatten(), expected_matrix.flatten()) print( f"mean absolute difference between actual/expected contact matrix for {code}/{option} is {str(round(d, 3))}" ) if d > 1: passed = passed & False return checked & passed
def get_pop_details(self, pop, dir, title_prefix, location, state_location, country_location, decimal=3): os.makedirs(dir, exist_ok=True) for setting_code in ['H', 'W', 'S']: average_contacts = utilities.get_average_contact_by_age( pop, self.datadir, setting_code=setting_code, decimal=decimal) fmt = f'%.{str(decimal)}f' # print(f"expected contacts by age for {code}:\n", average_contacts) utilities.plot_array( average_contacts, datadir=self.figDir, testprefix= f"{self.n}_seed_{self.seed}_{setting_code}_average_contacts", expect_label='Expected' if self.generateBaseline else 'Test') sc.savejson(os.path.join( dir, f"{self.n}_seed_{self.seed}_{setting_code}_average_contact.json" ), dict(enumerate(average_contacts.tolist())), indent=2) for type in ['density', 'frequency']: matrix = sp.calculate_contact_matrix(pop, type, setting_code) brackets = sp.get_census_age_brackets(self.datadir, state_location, country_location) ageindex = sp.get_age_by_brackets_dic(brackets) agg_matrix = sp.get_aggregate_matrix(matrix, ageindex) np.savetxt(os.path.join( dir, f"{self.n}_seed_{self.seed}_{setting_code}_{type}_contact_matrix.csv" ), agg_matrix, delimiter=",", fmt=fmt) fig = plot_age_mixing_matrices.test_plot_generated_contact_matrix( setting_code=setting_code, population=pop, title_prefix=" Expected " if self.generateBaseline else " Test ", density_or_frequency=type) # fig.show() fig.savefig( os.path.join( self.figDir, f"{self.n}_seed_{self.seed}_{setting_code}_{type}_contact_matrix.png" ))
def test_plot_generated_trimmed_contact_matrix(setting_code='H', n=5000, aggregate_flag=True, logcolors_flag=True, density_or_frequency='density', with_facilities=False, cmap='cmr.freeze_r', fontsize=16, rotation=50): """ Plot the age mixing matrix for a specific setting where the edges are trimmed. Args: setting_code (str) : name of the physial contact setting: H for households, S for schools, W for workplaces, C for community or other n (int) : number of people in the population aggregate_flag (book) : If True, plot the contact matrix for aggregate age brackets, else single year age contact matrix. logcolors_flag (bool) : If True, plot heatmap in logscale density_or_frequency (str) : If 'density', then each contact counts for 1/(group size -1) of a person's contact in a group, elif 'frequency' then count each contact. This means that more people in a group leads to higher rates of contact/exposure. with_facilities (bool) : If True, create long term care facilities cmap(str or matplotlib colormap) : colormap fontsize (int) : base font size rotation (int) : rotation for x axis labels Returns: A fig object. """ datadir = sp.datadir state_location = 'Washington' location = 'seattle_metro' country_location = 'usa' # popdict = {} options_args = {'use_microstructure': True} network_distr_args = {'Npop': int(n)} # contacts = sp.make_contacts(popdict, state_location=state_location, location=location, options_args=options_args, # network_distr_args=network_distr_args) # contacts = sp.trim_contacts(contacts, trimmed_size_dic=None, use_clusters=False) population = sp.make_population(n, generate=True, with_facilities=with_facilities) age_brackets = sp.get_census_age_brackets(datadir, state_location=state_location, country_location=country_location) age_by_brackets_dic = sp.get_age_by_brackets_dic(age_brackets) ages = [] for uid in population: ages.append(population[uid]['age']) age_count = Counter(ages) aggregate_age_count = sp.get_aggregate_ages(age_count, age_by_brackets_dic) matrix = sp.calculate_contact_matrix(population, density_or_frequency, setting_code) fig = sp.plot_contact_matrix(matrix, age_count, aggregate_age_count, age_brackets, age_by_brackets_dic, setting_code, density_or_frequency, logcolors_flag, aggregate_flag, cmap, fontsize, rotation) return fig
def get_pop_contact_matrix(pop, layer, method): """ Args: pop (pop object) : population, either synthpops.pop.Pop, covasim.people.People, or dict layer (str) : name of the physical contact layer: H for households, S for schools, W for workplaces, C for community or other method: (str) : density or frequency Returns: array: contact matrix """ age_brackets = spdd.get_census_age_brackets(**pop.loc_pars) matrix = sp.calculate_contact_matrix(pop.popdict, method, layer) ageindex = spb.get_age_by_brackets(age_brackets) agg_matrix = spb.get_aggregate_matrix(matrix, ageindex) return agg_matrix
def test_older_ages_have_household_contacts(): """ Test that older age groups (85+) have at least some household contacts with other older individuals if expected. Together, if sp.Pop.generate() uses the incorrect number of age brackets with the contact matrices, older age groups will not be generated as household contacts for each other (when we look at the generated contact matrix for households, the blocks between 85+ year olds would then be 0 for relatively large populations, even though the household contact matrix would have us expect otherwise.) """ test_pars = sc.dcp(pars) test_pars['n'] = 20e3 pop = sp.Pop(**test_pars) pop_dict = pop.to_dict() contact_matrix_dic = sp.get_contact_matrix_dic(sp.datadir, sheet_name=pop.sheet_name) contact_matrix_nbrackets = contact_matrix_dic[list( contact_matrix_dic.keys())[0]].shape[0] cm_age_brackets = sp.get_census_age_brackets( sp.datadir, country_location=pop.country_location, state_location=pop.state_location, location=pop.location, nbrackets=contact_matrix_nbrackets) cm_age_by_brackets_dic = sp.get_age_by_brackets_dic(cm_age_brackets) age_threshold = 85 age_threshold_bracket = cm_age_by_brackets_dic[age_threshold] expected_older_contact = np.sum( contact_matrix_dic['H'][age_threshold_bracket:, age_threshold_bracket:]) matrix = sp.calculate_contact_matrix(pop_dict, setting_code='H') gen_older_age_contacts = np.sum(matrix[age_threshold:, age_threshold:]) if expected_older_contact != 0: assert gen_older_age_contacts != 0, f'Check failed, individuals over {age_threshold} years old have no contacts with each other in households even though the household contact matrix expects them to.' else: assert gen_older_age_contacts == 0, f'Check failed, individuals over {age_threshold} years old have {gen_older_age_contacts} household contacts with each other even though the household contact matrix expects them to have none.' print('Check passed.')
def test_older_ages_have_household_contacts(): """ Test that older age groups (80+) have at least some household contacts with other older individuals if expected. Together, if sp.Pop.generate() uses the incorrect number of age brackets with the contact matrices, older age groups will not be generated as household contacts for each other (when we look at the generated contact matrix for households, the blocks between 85+ year olds would then be 0 for relatively large populations, even though the household contact matrix would have us expect otherwise.) """ test_pars = sc.dcp(pars) test_pars.n = settings.pop_sizes.medium_large # decent size to check older populations in households pop = sp.Pop(**test_pars) pop_dict = pop.to_dict() loc_pars = pop.loc_pars contact_matrices = sp.get_contact_matrices(sp.settings.datadir, sheet_name=pop.sheet_name) contact_matrix_nbrackets = contact_matrices[list( contact_matrices.keys())[0]].shape[0] cm_age_brackets = sp.get_census_age_brackets( **sc.mergedicts(loc_pars, {'nbrackets': contact_matrix_nbrackets})) cm_age_by_brackets = sp.get_age_by_brackets(cm_age_brackets) age_threshold = 80 age_threshold_bracket = cm_age_by_brackets[age_threshold] expected_older_contact = np.sum( contact_matrices['H'][age_threshold_bracket:, age_threshold_bracket:]) matrix = sp.calculate_contact_matrix(pop_dict, layer='H') gen_older_age_contacts = np.sum(matrix[age_threshold:, age_threshold:]) if expected_older_contact != 0: assert gen_older_age_contacts != 0, f'Check failed, individuals over {age_threshold} years old have no contacts with each other in households even though the household contact matrix expects them to.' else: assert gen_older_age_contacts == 0, f'Check failed, individuals over {age_threshold} years old have {gen_older_age_contacts} household contacts with each other even though the household contact matrix expects them to have none.' print('Check passed.')
vbounds['S'] = {'vmin': 1e-2, 'vmax': 1e0} vbounds['W'] = {'vmin': 1e-2, 'vmax': 1e0} vbounds['LTCF'] = {'vmin': 1e-2, 'vmax': 1e-0} im = [] cax = [] cbar = [] rotation = 66 for l, layer in enumerate(keys_to_plot): setting_code = layer.title() if setting_code == 'L': setting_code = 'LTCF' print(f'Plotting average age mixing contact matrix in layer: {layer}') matrix_dic[layer] = sp.calculate_contact_matrix(population, density_or_frequency, setting_code) if aggregate_flag: aggregate_matrix = sp.get_aggregate_matrix(matrix_dic[layer], age_by_brackets_dic) asymmetric_matrix = sp.get_asymmetric_matrix(aggregate_matrix, aggregate_age_count) else: asymmetric_matrix = sp.get_asymmetric_matrix(matrix_dic[layer], age_count) im.append(ax[2 * l].imshow(asymmetric_matrix.T, origin='lower', interpolation='nearest',