def test_make_contacts_with_facilities_from_microstructure( location='seattle_metro', state_location='Washington', country_location='usa', n=1000): # First generate microstructure with facilities sp.make_population(n, datadir, location, state_location, country_location, write=False, plot=False, return_popdict=True) # sp.generate_microstructure_with_facilities(datadir, location, state_location, # country_location, n, sheet_name='United States of America', # use_two_group_reduction=False, average_LTCF_degree=20, ltcf_staff_age_min=20, ltcf_staff_age_max=60, # with_school_types=False, school_mixing_type='random', average_class_size=20, inter_grade_mixing=0.1, # average_student_teacher_ratio=20, average_teacher_teacher_degree=3, teacher_age_min=25, teacher_age_max=75, # average_student_all_staff_ratio=15, average_additional_staff_degree=20, staff_age_min=20, staff_age_max=75, # verbose=False, plot=False, write=False, return_popdict=False, use_default=False): # Then make contacts popdict = sp.make_population(n, datadir, location, state_location, country_location) contains_age = 74 contains_sex = 1 uids = popdict.keys() assert contains_age in uids assert contains_sex in uids return popdict
def test_make_population_with_multi_flags(): with pytest.raises(ValueError, match=r"Requesting both long term*") as info: sp.make_population(n=n, generate=True, with_industry_code=True, with_facilities=True) return info
def test_inter_grade_mixing(school_mixing_type='random'): """ When school_mixing_type is 'age_clustered' then inter_grade_mixing should rewire a fraction of the edges between students in the same age or grade to be edges with any other student in the school. When school_mixing_type is 'random' or 'age_and_class_clustered', inter_grade_mixing has no effect. Args: school_mixing_type (str): The mixing type for schools, 'random', 'age_clustered', or 'age_and_class_clustered'. """ sp.logger.info( f'Testing the effect of the parameter inter_grade_mixing for school_mixing_type: {school_mixing_type}' ) test_pars = sc.dcp(pars) test_pars['school_mixing_type'] = school_mixing_type pop_1 = sp.make_population(**test_pars) test_pars['inter_grade_mixing'] = 0.3 pop_2 = sp.make_population(**test_pars) # make an adjacency matrix of edges between students adjm_1 = np.zeros((101, 101)) adjm_2 = np.zeros((101, 101)) student_ids = set() for i, person in pop_1.items(): if person['sc_student']: student_ids.add(i) for ni, i in enumerate(student_ids): contacts_1 = pop_1[i]['contacts']['S'] contacts_2 = pop_2[i]['contacts']['S'] student_contacts_1 = list(set(contacts_1).intersection(student_ids)) student_contacts_2 = list(set(contacts_2).intersection(student_ids)) age_i = pop_1[i]['age'] for nj, j in enumerate(student_contacts_1): age_j = pop_1[j]['age'] adjm_1[age_i][age_j] += 1 for nj, j in enumerate(student_contacts_2): age_j = pop_2[j]['age'] adjm_2[age_i][age_j] += 1 if school_mixing_type in ['random', 'age_and_class_clustered']: assert np.not_equal(adjm_1, adjm_2).sum( ) == 0, f"inter_grade_mixing parameter check failed. Different values for this parameter produced different results for school_mixing_type: {school_mixing_type}." elif school_mixing_type in ['age_clustered']: assert np.not_equal(adjm_1, adjm_2).sum( ) > 0, f"inter_grade_mixing parameter check failed. It produced the same results for different values of this parameter for school_mixing_type: {school_mixing_type}."
def test_api(): n = 50000 max_contacts = {'S': 20, 'W': 10} population = sp.make_population(n=n, max_contacts=max_contacts) with pytest.raises(ValueError): population = sp.make_population(n=298437) # Not a supported number return population
def test_sizes(): ''' Synthpops should support populations down to 200 people ''' nlist = [200, 201, 1001, 9848] for n in nlist: print(f'Working on {n}') pop = sp.make_population(n=n, generate=True) assert len(pop) == n with pytest.raises(NotImplementedError): sp.make_population(n=199, generate=True) # Too small return pop
def test_api(): ''' Basic SynthPops test ''' sp.logger.info('Testing API and logger') n = 2000 max_contacts = {'S': 20, 'W': 10} population = sp.make_population(n=n, max_contacts=max_contacts) with pytest.raises(ValueError): population = sp.make_population( n=298437, generate=False) # Not a supported number return population
def make_pop(): n = [ 10000, 10001 ][1] # Use either a pre-generated population, or one that has to be made from scratch max_contacts = {'S': 20, 'W': 10} population = sp.make_population(n=n, max_contacts=max_contacts) return population
def make_synthpop(sim): ''' Make a population using synthpops, including contacts ''' import synthpops as sp # Optional import population = sp.make_population(n=sim['pop_size']) uids, ages, sexes, contacts = [], [], [], [] for uid,person in population.items(): uids.append(uid) ages.append(person['age']) sexes.append(person['sex']) # Replace contact UIDs with ints... uid_mapping = {uid:u for u,uid in enumerate(uids)} key_mapping = {'H':'h', 'S':'s', 'W':'w', 'C':'c'} # Remap keys from old names to new names for uid,person in population.items(): uid_contacts = person['contacts'] int_contacts = {} for key in uid_contacts.keys(): new_key = key_mapping[key] int_contacts[new_key] = [] for uid in uid_contacts[key]: int_contacts[new_key].append(uid_mapping[uid]) int_contacts[new_key] = np.array(int_contacts[new_key], dtype=np.int32) contacts.append(int_contacts) popdict = {} popdict['uid'] = uids popdict['age'] = np.array(ages) popdict['sex'] = np.array(sexes) popdict['contacts'] = contacts popdict['layer_keys'] = list(key_mapping.values()) return popdict
def test_make_popdict_generic(n=default_n): sc.heading(f'Making popdict for {n} people') n = int(n) popdict = sp.make_population( n=n, use_demography=False) # Non-USA not implemented return popdict
def make_people(sim, verbose=None, id_len=None, die=True): # Set defaults if verbose is None: verbose = sim['verbose'] if id_len is None: id_len = 6 # Set inputs n_people = int(sim['n']) # Shorten usepopdata = sim['usepopdata'] # Shorten use_rand_pop = ( usepopdata == 'random' ) # Whether or not to use a random population (as opposed to synthpops) # Check which type of population to rpoduce if not use_rand_pop and not cvreqs.available['synthpops']: errormsg = f'You have requested "{usepopdata}" population, but synthpops is not available; please use "random"' if die: raise ValueError(errormsg) else: print(errormsg) usepopdata = 'random' # Actually create the population if use_rand_pop: popdict = make_randpop(sim) else: import synthpops as sp # Optional import popdict = sp.make_population(n=sim['n']) # Set prognoses by modifying popdict in place set_prognoses(sim, popdict) # Actually create the people people = { } # Dictionary for storing the people -- use plain dict since faster than odict for p in range(n_people): # Loop over each person keys = [ 'uid', 'age', 'sex', 'contacts', 'symp_prob', 'severe_prob', 'crit_prob', 'death_prob' ] person_args = {} for key in keys: person_args[key] = popdict[key][p] # Convert from list to dict person = Person(pars=sim.pars, **person_args) # Create the person people[person_args['uid']] = person # Save them to the dictionary # Store UIDs and people sim.uids = popdict['uid'] sim.people = people sim.contact_keys = list(sim['contacts_pop'].keys()) average_age = sum(popdict['age'] / n_people) sc.printv( f'Created {n_people} people, average age {average_age:0.2f} years', 1, verbose) return
def test_Spokane(): """Test that a Dakar population can be created with the basic SynthPops API.""" sp.logger.info("Test that a Spokane population can be created with the basic SynthPops API.") pop = sp.make_population(**pars) age_distr = sp.read_age_bracket_distr(sp.datadir, country_location='usa', state_location='Washington', location='seattle_metro') assert len(age_distr) == 20, f'Check failed, len(age_distr): {len(age_distr)}' # will remove if this passes in github actions test sp.set_location_defaults('defaults') # Reset default values after this test is complete. return pop
def make_pop(n): """ run function to create n population for input n """ # n = [10000, 10001][1] # Use either a pre-generated population, or one that has to be made from scratch max_contacts = { 'W': 10 } # only workplaces get trimmed. Schools have a separate method in the schools_module population = sp.make_population(n=n, max_contacts=max_contacts) return population
def test_api(): ''' Basic SynthPops test ''' sp.logger.info('Testing API and logger') n = 2000 max_contacts = {'S': 20, 'W': 10} population = sp.make_population(n=n, max_contacts=max_contacts) return population
def test_sizes(): ''' Synthpops should support populations down to 200 people ''' nlist = [100, 246] for n in nlist: print(f'Working on {n}') pop = sp.make_population(n=n, generate=True) assert len(pop) == n return pop
def test_generate_microstructures_with_facilities(): popdict = sp.make_population(datadir=datadir, location=location, state_location=state_location, country_location=country_location, n=n, write=write, plot=False, return_popdict=return_popdict) assert (len(popdict) is not None) return popdict
def test_without_school_types(): """ Test that without school types, all schools are put together in one group. """ sp.logger.info("Creating schools where with_school_types is False.") test_pars = sc.dcp(pars) test_pars['with_school_types'] = None pop = sp.make_population(**test_pars) fig, ax, school_types = plot_school_sizes_by_type(pop, test_pars, do_show=True) return pop, fig, ax, school_types
def test_plot_generated_trimmed_contact_matrix(setting_code='H', n=5000, aggregate_flag=True, logcolors_flag=True, density_or_frequency='density', with_facilities=False, cmap='cmr.freeze_r', fontsize=16, rotation=50): """ Plot the age mixing matrix for a specific setting where the edges are trimmed. Args: setting_code (str) : name of the physial contact setting: H for households, S for schools, W for workplaces, C for community or other n (int) : number of people in the population aggregate_flag (book) : If True, plot the contact matrix for aggregate age brackets, else single year age contact matrix. logcolors_flag (bool) : If True, plot heatmap in logscale density_or_frequency (str) : If 'density', then each contact counts for 1/(group size -1) of a person's contact in a group, elif 'frequency' then count each contact. This means that more people in a group leads to higher rates of contact/exposure. with_facilities (bool) : If True, create long term care facilities cmap(str or matplotlib colormap) : colormap fontsize (int) : base font size rotation (int) : rotation for x axis labels Returns: A fig object. """ datadir = sp.datadir state_location = 'Washington' location = 'seattle_metro' country_location = 'usa' # popdict = {} options_args = {'use_microstructure': True} network_distr_args = {'Npop': int(n)} # contacts = sp.make_contacts(popdict, state_location=state_location, location=location, options_args=options_args, # network_distr_args=network_distr_args) # contacts = sp.trim_contacts(contacts, trimmed_size_dic=None, use_clusters=False) population = sp.make_population(n, generate=True, with_facilities=with_facilities) age_brackets = sp.get_census_age_brackets(datadir, state_location=state_location, country_location=country_location) age_by_brackets_dic = sp.get_age_by_brackets_dic(age_brackets) ages = [] for uid in population: ages.append(population[uid]['age']) age_count = Counter(ages) aggregate_age_count = sp.get_aggregate_ages(age_count, age_by_brackets_dic) matrix = sp.calculate_contact_matrix(population, density_or_frequency, setting_code) fig = sp.plot_contact_matrix(matrix, age_count, aggregate_age_count, age_brackets, age_by_brackets_dic, setting_code, density_or_frequency, logcolors_flag, aggregate_flag, cmap, fontsize, rotation) return fig
def make_synthpop(sim, directed=False): ''' Make a population using synthpops, including contacts ''' import synthpops as sp # Optional import pop_size = sim['pop_size'] population = sp.make_population(n=pop_size, generate=True) uids, ages, sexes, contacts = [], [], [], [] for uid, person in population.items(): uids.append(uid) ages.append(person['age']) sexes.append(person['sex']) # Replace contact UIDs with ints uid_mapping = {uid: u for u, uid in enumerate(uids)} key_mapping = { 'H': 'h', 'S': 's', 'W': 'w', 'C': 'c' } # Remap keys from old names to new names for uid in uids: iid = uid_mapping[uid] # Integer UID person = population.pop(uid) uid_contacts = sc.dcp(person['contacts']) int_contacts = {} for spkey in uid_contacts.keys(): lkey = key_mapping[ spkey] # Map the SynthPops key into a Covasim layer key int_contacts[lkey] = [] for cid in uid_contacts[spkey]: # Contact ID icid = uid_mapping[cid] # Integer contact ID if icid > iid or directed: # Don't add duplicate contacts int_contacts[lkey].append(icid) int_contacts[lkey] = np.array(int_contacts[lkey], dtype=cvd.default_int) contacts.append(int_contacts) # Add community contacts c_contacts, _ = make_random_contacts(pop_size, {'c': sim['contacts']['c']}) for i in range(pop_size): contacts[i]['c'] = c_contacts[i][ 'c'] # Copy over community contacts -- present for everyone # Finalize popdict = {} popdict['uid'] = sc.dcp(uids) popdict['age'] = np.array(ages) popdict['sex'] = np.array(sexes) popdict['contacts'] = sc.dcp(contacts) layer_keys = list(key_mapping.values()) return popdict, layer_keys
def test_with_non_teaching_staff(): """ When with_non_teaching_staff is True, each school should be created with non teaching staff. Otherwise when False, there should be no non teaching staff in schools. """ sp.logger.info( f'Testing the effect of the parameter with_non_teaching_staff.') test_pars = sc.dcp(pars) test_pars['with_non_teaching_staff'] = True pop_1 = sp.make_population(**test_pars) test_pars['with_non_teaching_staff'] = False pop_2 = sp.make_population(**test_pars) school_staff_1 = {} for i, person in pop_1.items(): if person['scid'] is not None: school_staff_1.setdefault(person['scid'], 0) if person['sc_staff']: school_staff_1[person['scid']] += 1 staff_ids_2 = set() for i, person in pop_2.items(): if person['sc_staff']: staff_ids_2.add(i) # with_non_teaching_staff == True so minimum number of staff per school needs to be 1 assert min( school_staff_1.values() ) >= 1, f"with_non_teaching_staff parameter check failed when set to True." # with_non_teaching_staff == False so there should be no one who shows up with sc_staff set to 1 assert len( staff_ids_2 ) == 0, f"with_non_teaching_staff parameter check failed when set to False."
def test_original_household_method(do_show=False): sp.logger.info("Generating households with the infer_ages method.") test_pars = sc.dcp(pars) test_pars['household_method'] = 'infer_ages' pop = sp.make_population(**test_pars) datadir = sp.datadir fig, ax = plot_age_dist(datadir, pop, test_pars, do_show, test_pars['household_method']) if do_show: plt.show() return pop
def test_Dakar(): """Test that a Dakar population can be created with the basic SynthPops API.""" sp.logger.info("Test that a Dakar population can be created with the basic SynthPops API.") sp.set_nbrackets(18) # Dakar age distributions available are up to 18 age brackets pop = sp.make_population(**pars) assert len(pop) == pars['n'], 'Check failed.' print('Check passed') sp.set_location_defaults('defaults') # Reset default values after this test is complete. sp.logger.info("Test that the default country was reset.") assert sp.default_country == 'usa', f'Check failed: default_country is {sp.default_country}' print('2nd Check passed') return pop
def test_make_population_with_industry_code(): popdict = sp.make_population(n=n, generate=True, with_industry_code=True, use_two_group_reduction=True, average_LTCF_degree=20) # verify these keys are either None or set to a value for i, uid in enumerate(popdict): if popdict[uid]['wpid'] is None: assert popdict[uid]['wpid'] is None else: assert popdict[uid]['wpid'] is not None if popdict[uid]['wpindcode'] is None: assert popdict[uid]['wpindcode'] is None else: assert popdict[uid]['wpindcode'] is not None return popdict
def test_basic_api(): ''' Basic SynthPops test ''' sp.logger.info('Testing basic API') pop = sp.make_population(**pars) if regenerate or not os.path.exists(outfile): print('Saving...') sc.saveobj(outfile, pop) else: print('Checking...') pop2 = sc.loadobj(outfile) assert pop == pop2, 'Check failed' print('Check passed') return pop
def make_synthpop(sim): ''' Make a population using synthpops, including contacts ''' import synthpops as sp # Optional import pop_size = sim['pop_size'] population = sp.make_population(n=pop_size) uids, ages, sexes, contacts = [], [], [], [] for uid, person in population.items(): uids.append(uid) ages.append(person['age']) sexes.append(person['sex']) # Replace contact UIDs with ints... uid_mapping = {uid: u for u, uid in enumerate(uids)} key_mapping = { 'H': 'h', 'S': 's', 'W': 'w', 'C': 'c' } # Remap keys from old names to new names for uid in uids: person = population.pop(uid) uid_contacts = sc.dcp(person['contacts']) int_contacts = {} for key in uid_contacts.keys(): new_key = key_mapping[key] int_contacts[new_key] = [] for uid in uid_contacts[key]: int_contacts[new_key].append(uid_mapping[uid]) int_contacts[new_key] = np.array(int_contacts[new_key], dtype=cvd.default_int) contacts.append(int_contacts) # Add community contacts c_contacts, _ = make_random_contacts(pop_size, {'c': sim['contacts']['c']}) for i in range(pop_size): contacts[i]['c'] = c_contacts[i][ 'c'] # Copy over community contacts -- present for everyone # Finalize popdict = {} popdict['uid'] = sc.dcp(uids) popdict['age'] = np.array(ages) popdict['sex'] = np.array(sexes) popdict['contacts'] = sc.dcp(contacts) layer_keys = list(key_mapping.values()) return popdict, layer_keys
def runpop(self, filename, actual_vals, testprefix = "test"): # if default sort order is not concerned: # pop = dict(sorted(pop.items(), key=lambda x: x[0])) params = {} args = inspect.getfullargspec(sp.make_population).args defaults = inspect.getfullargspec(sp.make_population).defaults for i in range(0, len(args)): params[args[i]] = defaults[i] for name in actual_vals: if name in params.keys(): params[name] = actual_vals[name] with open(os.path.join(self.configDir, f"{testprefix}.txt"), mode="w") as cf: for key, value in params.items(): cf.writelines(str(key) + ':' + str(value) + "\n") pop = sp.make_population(**params) return pop
def test_school_types_created(): """ Test that unique school types are created. Returns: A list of the school types expected for the specified location. """ sp.logger.info( f"Test that unique school types are created for each school.\nRun this first to see what school types you are working with." ) test_pars = sc.dcp(pars) test_pars['n'] = 20e3 pop = sp.make_population(**test_pars) if pars['with_school_types']: expected_school_size_distr = sp.get_school_size_distr_by_type( sp.datadir, location=pars['location'], state_location=pars['state_location'], country_location=pars['country_location'], use_default=pars['use_default']) expected_school_types = sorted(expected_school_size_distr.keys()) else: expected_school_types = [None] schools_by_type = dict() for i, person in pop.items(): if person['scid'] is not None: schools_by_type.setdefault(person['scid'], set()) schools_by_type[person['scid']].add(person['sc_type']) for s, school_type in schools_by_type.items(): assert len( school_type ) == 1, f'Check failed. School {s} is listed as having more than one type.' schools_by_type[s] = list(school_type)[0] gen_school_types = sorted(set(schools_by_type.values())) assert gen_school_types == expected_school_types, f"Check failed. generated types: {gen_school_types}, expected: {expected_school_types}" print( f"School types generated for {test_pars['location']}: {set(schools_by_type.values())}" ) return list(set(schools_by_type.values()))
def test_make_population(): population = sp.make_population(n=n, generate=True, with_facilities=True, use_two_group_reduction=True, average_LTCF_degree=20) for key, person in population.items(): assert population[key]['contacts'] is not None assert population[key]['contacts']['LTCF'] is not None assert len(population[key]['contacts']['H']) >= 0 expected_layers = {'H', 'S', 'W', 'C', 'LTCF'} for layerkey in population[key]['contacts'].keys(): if layerkey in expected_layers: assert True else: assert False return population
def test_school_sizes_by_type(pars, do_show=False): """ Visually show how the school size distribution generated compares to the data for the location being simulated. Notes: The larger the population size, the better the generated school size distributions by school type can match the expected data. If generated populations are too small, larger schools will be missed and in general there won't be enough schools generated to apply statistical tests. """ sp.logger.info( "Creating schools by school type and a visual comparison of how they match to data." ) pop = sp.make_population(**pars) fig, ax, school_types = plot_school_sizes_by_type(pop, pars, do_show=True) return pop, fig, ax, school_types
def test_smoothed_and_fixed_ages_household_method(do_show=False): sp.logger.info( "Generating households with the fixed_ages and smoothed_ages methods.") test_pars = sc.dcp(pars) test_pars['location'] = 'Spokane_County' test_pars['household_method'] = 'fixed_ages' test_pars['smooth_ages'] = True test_pars['window_length'] = 7 # window for averaging the age distribution pop = sp.make_population(**test_pars) datadir = sp.datadir fig, ax = plot_age_dist(datadir, pop, test_pars, do_show, test_pars['household_method']) if do_show: plt.show() return fig, ax
def test_make_population(location='seattle_metro', state_location='Washington', n=10000): contacts = sp.make_population(datadir=datadir, location=location, state_location=state_location, country_location=country_location, n=n, with_industry_code=False) uids = contacts.keys() uids = [uid for uid in uids] for n, uid in enumerate(uids): if n > 20: break layers = contacts[uid]['contacts'] print('uid', uid, 'age', contacts[uid]['age'], 'total contacts', np.sum([len(contacts[uid]['contacts'][k]) for k in layers])) for k in layers: contact_ages = [ contacts[c]['age'] for c in contacts[uid]['contacts'][k] ] print(k, len(contact_ages), 'contact ages', contact_ages) schools = contacts[uid]['contacts']['S'] school_id = contacts[uid]['scid'] school_teacher = contacts[uid]['sc_teacher'] school_student = contacts[uid]['sc_student'] assert schools is not None if school_id is None: assert school_id is None else: assert school_id is not None if school_teacher is None: assert school_teacher is None else: assert school_teacher == 1 if school_student is None: assert school_student is None else: assert school_student == 1 return contacts