def test_readPopulationAgeStructured_fails_missing_header(): with tempfile.NamedTemporaryFile(mode="w+", delete=False) as fp: rows = ["S08000015,Female,12,2,3,5", "S08000015,Male,20,7,11,13"] fp.write("\n".join(rows)) fp.flush() with pytest.raises(ValueError): loaders.readPopulationAgeStructured(fp.name)
def test_readPopulationAgeStructured_fails_inconsistent_data(): with tempfile.NamedTemporaryFile(mode="w+", delete=False) as fp: rows = [ "Health_Board,Sex,Total_across_age,Young,Medium,Old", "S08000015,Female,12,2,3,5", "S08000015,Male,20,7,11,13", ] fp.write("\n".join(rows)) fp.flush() with pytest.raises(ValueError): loaders.readPopulationAgeStructured(fp.name)
def test_readPopulationAgeStructured_bad_total(total): df = pd.DataFrame([ { "Health_Board": "S08000015", "Sex": "Female", "Age": "[17,70)", "Total": total }, ]) with pytest.raises(ValueError): loaders.readPopulationAgeStructured(df)
def test_readPopulationAgeStructured_single_row(): with tempfile.NamedTemporaryFile(mode="w+", delete=False) as fp: rows = ["Health_Board,Sex,Total_across_age,Young,Medium,Old", "S08000015,Female,10,2,3,5"] fp.write("\n".join(rows)) fp.flush() population = loaders.readPopulationAgeStructured(fp.name) assert population["S08000015"]["Female"] == {"y": 2, "m": 3, "o": 5, "All_Ages": 10} assert population["S08000015"]["All_Sex"]["y"] == 2 assert population["S08000015"]["All_Sex"]["m"] == 3 assert population["S08000015"]["All_Sex"]["o"] == 5 assert population["S08000015"]["All_Sex"]["All_Ages"] == 10
def test_readPopulationAgeStructured_sums_all_ages(): with tempfile.NamedTemporaryFile(mode="w+", delete=False) as fp: rows = [ "Health_Board,Sex,Total_across_age,Young,Medium,Old", "S08000015,Female,10,2,3,5", "S08000015,Male,31,7,11,13", ] fp.write("\n".join(rows)) fp.flush() population = loaders.readPopulationAgeStructured(fp.name) assert population["S08000015"]["All_Sex"]["y"] == 2 + 7 assert population["S08000015"]["All_Sex"]["m"] == 3 + 11 assert population["S08000015"]["All_Sex"]["o"] == 5 + 13 assert population["S08000015"]["All_Sex"]["All_Ages"] == 10 + 31
def test_readPopulationAgeStructured_aggregate_ages(): df = pd.DataFrame([ { "Health_Board": "S08000015", "Sex": "Female", "Age": "[17,70)", "Total": 100 }, { "Health_Board": "S08000015", "Sex": "Male", "Age": "[17,70)", "Total": 100 }, ]) population = loaders.readPopulationAgeStructured(df) assert population == {"S08000015": {"[17,70)": 200}}
def test_basicSimulationInternalAgeStructure_invariants( age_transitions, demographics, commute_moves, compartment_names, age_infection_matrix, num_infected, generic_infection, seed, ): age_to_trans = np.setUpParametersAges( loaders.readParametersAgeStructured(age_transitions)) population = loaders.readPopulationAgeStructured(demographics) graph = loaders.genGraphFromContactFile(commute_moves) states = np.setupInternalPopulations(graph, compartment_names, list(age_to_trans.keys()), population) old_graph = copy.deepcopy(graph) old_age_to_trans = copy.deepcopy(age_to_trans) initial_population = sum(_count_people_per_region( states[0])) + num_infected np.basicSimulationInternalAgeStructure( rand=random.Random(seed), graph=graph, numInfected=num_infected, timeHorizon=50, genericInfection=generic_infection, ageInfectionMatrix=age_infection_matrix, diseaseProgressionProbs=age_to_trans, dictOfStates=states, ) # population remains constant assert all([ sum(_count_people_per_region(state)) == pytest.approx( initial_population) for state in states.values() ]) # the graph is unchanged assert nx.is_isomorphic(old_graph, graph) # infection matrix is unchanged assert age_to_trans == old_age_to_trans
def test_readPopulationAgeStructured(data_api): population = loaders.readPopulationAgeStructured( data_api.read_table("human/population")) expected = { "S08000015": { "[0,17)": 65307, "[17,70)": 245680, "70+": 58683 }, "S08000016": { "[0,17)": 20237, "[17,70)": 75008, "70+": 20025 }, "S08000017": { "[0,17)": 24842, "[17,70)": 96899, "70+": 27049 }, "S08000019": { "[0,17)": 55873, "[17,70)": 209221, "70+": 40976 }, "S08000020": { "[0,17)": 105607, "[17,70)": 404810, "70+": 74133 }, "S08000022": { "[0,17)": 55711, "[17,70)": 214008, "70+": 52081 }, "S08000024": { "[0,17)": 159238, "[17,70)": 635249, "70+": 103283 }, "S08000025": { "[0,17)": 3773, "[17,70)": 14707, "70+": 3710 }, "S08000026": { "[0,17)": 4448, "[17,70)": 15374, "70+": 3168 }, "S08000028": { "[0,17)": 4586, "[17,70)": 17367, "70+": 4877 }, "S08000029": { "[0,17)": 68150, "[17,70)": 250133, "70+": 53627 }, "S08000030": { "[0,17)": 71822, "[17,70)": 280547, "70+": 63711 }, "S08000031": { "[0,17)": 208091, "[17,70)": 829574, "70+": 137315 }, "S08000032": { "[0,17)": 125287, "[17,70)": 450850, "70+": 83063 }, } assert population == expected
def test_readPopulationAgeStructured(demographics): population = loaders.readPopulationAgeStructured(demographics) expected = { "S08000015": { "Female": {"y": 31950, "m": 127574, "o": 32930, "All_Ages": 192454}, "Male": {"y": 33357, "m": 118106, "o": 25753, "All_Ages": 177216}, "All_Sex": {"y": 65307, "m": 245680, "o": 58683, "All_Ages": 369670}, }, "S08000016": { "Female": {"y": 9957, "m": 38363, "o": 10961, "All_Ages": 59281}, "Male": {"y": 10280, "m": 36645, "o": 9064, "All_Ages": 55989}, "All_Sex": {"y": 20237, "m": 75008, "o": 20025, "All_Ages": 115270}, }, "S08000017": { "Female": {"y": 12176, "m": 49559, "o": 14734, "All_Ages": 76469}, "Male": {"y": 12666, "m": 47340, "o": 12315, "All_Ages": 72321}, "All_Sex": {"y": 24842, "m": 96899, "o": 27049, "All_Ages": 148790}, }, "S08000019": { "Female": {"y": 27102, "m": 106902, "o": 22899, "All_Ages": 156903}, "Male": {"y": 28771, "m": 102319, "o": 18077, "All_Ages": 149167}, "All_Sex": {"y": 55873, "m": 209221, "o": 40976, "All_Ages": 306070}, }, "S08000020": { "Female": {"y": 51441, "m": 201015, "o": 41366, "All_Ages": 293822}, "Male": {"y": 54166, "m": 203795, "o": 32767, "All_Ages": 290728}, "All_Sex": {"y": 105607, "m": 404810, "o": 74133, "All_Ages": 584550}, }, "S08000022": { "Female": {"y": 27187, "m": 107483, "o": 28823, "All_Ages": 163493}, "Male": {"y": 28524, "m": 106525, "o": 23258, "All_Ages": 158307}, "All_Sex": {"y": 55711, "m": 214008, "o": 52081, "All_Ages": 321800}, }, "S08000024": { "Female": {"y": 77589, "m": 324258, "o": 58984, "All_Ages": 460831}, "Male": {"y": 81649, "m": 310991, "o": 44299, "All_Ages": 436939}, "All_Sex": {"y": 159238, "m": 635249, "o": 103283, "All_Ages": 897770}, }, "S08000025": { "Female": {"y": 1847, "m": 7315, "o": 1989, "All_Ages": 11151}, "Male": {"y": 1926, "m": 7392, "o": 1721, "All_Ages": 11039}, "All_Sex": {"y": 3773, "m": 14707, "o": 3710, "All_Ages": 22190}, }, "S08000026": { "Female": {"y": 2112, "m": 7493, "o": 1672, "All_Ages": 11277}, "Male": {"y": 2336, "m": 7881, "o": 1496, "All_Ages": 11713}, "All_Sex": {"y": 4448, "m": 15374, "o": 3168, "All_Ages": 22990}, }, "S08000028": { "Female": {"y": 2204, "m": 8636, "o": 2743, "All_Ages": 13583}, "Male": {"y": 2382, "m": 8731, "o": 2134, "All_Ages": 13247}, "All_Sex": {"y": 4586, "m": 17367, "o": 4877, "All_Ages": 26830}, }, "S08000029": { "Female": {"y": 33155, "m": 128255, "o": 29880, "All_Ages": 191290}, "Male": {"y": 34995, "m": 121878, "o": 23747, "All_Ages": 180620}, "All_Sex": {"y": 68150, "m": 250133, "o": 53627, "All_Ages": 371910}, }, "S08000030": { "Female": {"y": 35016, "m": 142530, "o": 35848, "All_Ages": 213394}, "Male": {"y": 36806, "m": 138017, "o": 27863, "All_Ages": 202686}, "All_Sex": {"y": 71822, "m": 280547, "o": 63711, "All_Ages": 416080}, }, "S08000031": { "Female": {"y": 101619, "m": 423135, "o": 80469, "All_Ages": 605223}, "Male": {"y": 106472, "m": 406439, "o": 56846, "All_Ages": 569757}, "All_Sex": {"y": 208091, "m": 829574, "o": 137315, "All_Ages": 1174980}, }, "S08000032": { "Female": {"y": 61191, "m": 231416, "o": 47571, "All_Ages": 340178}, "Male": {"y": 64096, "m": 219434, "o": 35492, "All_Ages": 319022}, "All_Sex": {"y": 125287, "m": 450850, "o": 83063, "All_Ages": 659200}, }, } assert population == expected
def test_readPopulationAgeStructured_no_rows(): with tempfile.NamedTemporaryFile(mode="w+", delete=False) as fp: rows = ["Health_Board,Sex,Total_across_age,Young,Medium,Old"] fp.write("\n".join(rows)) fp.flush() assert loaders.readPopulationAgeStructured(fp.name) == {}
def test_readPopulationAgeStructured_invalid_file(): with pytest.raises(IOError): loaders.readPopulationAgeStructured("")
def test_readPopulationAgeStructured_empty_file(): with tempfile.NamedTemporaryFile(mode="w+", delete=False) as fp: with pytest.raises(ValueError): assert loaders.readPopulationAgeStructured(fp.name) == {}
def createNetworkOfPopulation( compartment_transition_table: pd.DataFrame, population_table: pd.DataFrame, commutes_table: pd.DataFrame, mixing_matrix_table: pd.DataFrame, infectious_states: pd.DataFrame, infection_prob: pd.DataFrame, initial_infections: pd.DataFrame, trials: pd.DataFrame, movement_multipliers_table: pd.DataFrame = None, stochastic_mode: pd.DataFrame = None, random_seed: pd.DataFrame = None) -> NetworkOfPopulation: """Create the network of the population, loading data from files. :param compartment_transition_table: pd.Dataframe specifying the transition rates between infected compartments. :param population_table: pd.Dataframe with the population size in each region by gender and age. :param commutes_table: pd.Dataframe with the movements between regions. :param mixing_matrix_table: pd.Dataframe with the age infection matrix. :param movement_multipliers_table: pd.Dataframe with the movement multipliers. This may be None, in which case no multipliers are applied to the movements. :param infectious_states: States that are considered infectious :param infection_prob: Probability that a given contact will result in an infection :param initial_infections: Initial infections of the population at time 0 :param trials: Number of trials for the model :param stochastic_mode: Use stochastic mode for the model :param random_seed: Random number generator seed used for stochastic mode :return: The constructed network """ infection_prob = loaders.readInfectionProbability(infection_prob) infectious_states = loaders.readInfectiousStates(infectious_states) initial_infections = loaders.readInitialInfections(initial_infections) trials = loaders.readTrials(trials) stochastic_mode = loaders.readStochasticMode(stochastic_mode) random_seed = loaders.readRandomSeed(random_seed) # diseases progression matrix progression = loaders.readCompartmentRatesByAge( compartment_transition_table) # population census data population = loaders.readPopulationAgeStructured(population_table) # Check some requirements for this particular model to work with the progression matrix all_states = set() for states in progression.values(): assert SUSCEPTIBLE_STATE not in states, "progression from susceptible state is not allowed" for state, nextStates in states.items(): for nextState in nextStates: all_states.add(state) all_states.add(nextState) assert state == nextState or nextState != EXPOSED_STATE, \ "progression into exposed state is not allowed other than in self reference" assert (set(infectious_states) - all_states) == set(), \ f"mismatched infectious states and states {infectious_states} {set(progression.keys())}" # people movement's graph graph = loaders.genGraphFromContactFile(commutes_table) # movement multipliers (dampening or heightening) if movement_multipliers_table is not None: movementMultipliers = loaders.readMovementMultipliers( movement_multipliers_table) else: movementMultipliers: Dict[int, loaders.Multiplier] = {} # age-based infection matrix mixingMatrix = loaders.MixingMatrix(mixing_matrix_table) agesInInfectionMatrix = set(mixingMatrix) for age in mixingMatrix: assert agesInInfectionMatrix == set( mixingMatrix[age]), "infection matrix columns/rows mismatch" # Checks across datasets assert agesInInfectionMatrix == set( progression.keys()), "infection matrix and progression ages mismatch" assert agesInInfectionMatrix == {age for region in population.values() for age in region}, \ "infection matrix and population ages mismatch" assert set(graph.nodes()) == set( population.keys()), "regions mismatch between graph and population" state0: Dict[str, Dict[Tuple[str, str], float]] = {} for node in list(graph.nodes()): region = state0.setdefault(node, {}) for age, compartments in progression.items(): region[(age, SUSCEPTIBLE_STATE)] = population[node][age] for compartment in compartments: region[(age, compartment)] = 0 logger.info("Nodes: %s, Ages: %s, States: %s", len(state0), agesInInfectionMatrix, all_states) return NetworkOfPopulation(progression=progression, graph=graph, initialState=state0, mixingMatrix=mixingMatrix, movementMultipliers=movementMultipliers, infectiousStates=infectious_states, infectionProb=infection_prob, initialInfections=initial_infections, trials=trials, stochastic=stochastic_mode, randomState=np.random.default_rng(random_seed))
def createNetworkOfPopulation( compartment_transition_table: pd.DataFrame, population_table: pd.DataFrame, commutes_table: pd.DataFrame, mixing_matrix_table: pd.DataFrame, infectious_states: pd.DataFrame, infection_prob: pd.DataFrame, initial_infections: pd.DataFrame, trials: pd.DataFrame, start_end_date: pd.DataFrame, movement_multipliers_table: pd.DataFrame = None, stochastic_mode: pd.DataFrame = None, ) -> Tuple[NetworkOfPopulation, List[standard_api.Issue]]: """Create the network of the population, loading data from files. :param compartment_transition_table: pd.Dataframe specifying the transition rates between infected compartments. :param population_table: pd.Dataframe with the population size in each region by gender and age. :param commutes_table: pd.Dataframe with the movements between regions. :param mixing_matrix_table: pd.Dataframe with the age infection matrix. :param infectious_states: States that are considered infectious :param infection_prob: Probability that a given contact will result in an infection :param initial_infections: Initial infections of the population at time 0 :param trials: Number of trials for the model :param start_end_date: Starting and ending dates of the model :param movement_multipliers_table: pd.Dataframe with the movement multipliers. This may be None, in which case no multipliers are applied to the movements. :param stochastic_mode: Use stochastic mode for the model :return: A tuple with the constructed network and a list of any issues found. Although it may look convenient, storing this list inside of NetworkOfPopulation is not a good idea, as that may give the functions that receive it the false impression they can just append new issues there """ issues: List[standard_api.Issue] = [] infection_prob = loaders.readInfectionProbability(infection_prob) infectious_states = loaders.readInfectiousStates(infectious_states) initial_infections = loaders.readInitialInfections(initial_infections) trials = loaders.readTrials(trials) start_date, end_date = loaders.readStartEndDate(start_end_date) stochastic_mode = loaders.readStochasticMode(stochastic_mode) # diseases progression matrix progression = loaders.readCompartmentRatesByAge( compartment_transition_table) # population census data population = loaders.readPopulationAgeStructured(population_table) # Check some requirements for this particular model to work with the progression matrix all_states = set() for states in progression.values(): assert SUSCEPTIBLE_STATE not in states, "progression from susceptible state is not allowed" for state, nextStates in states.items(): for nextState in nextStates: all_states.add(state) all_states.add(nextState) assert state == nextState or nextState != EXPOSED_STATE, \ "progression into exposed state is not allowed other than in self reference" assert (set(infectious_states) - all_states) == set(), \ f"mismatched infectious states and states {infectious_states} {all_states}" # people movement's graph graph = loaders.genGraphFromContactFile(commutes_table) # movement multipliers (dampening or heightening) if movement_multipliers_table is not None: movementMultipliers = loaders.readMovementMultipliers( movement_multipliers_table) else: movementMultipliers = {} # age-based infection matrix mixingMatrix = loaders.MixingMatrix(mixing_matrix_table) agesInInfectionMatrix = set(mixingMatrix) for age in mixingMatrix: assert agesInInfectionMatrix == set( mixingMatrix[age]), "infection matrix columns/rows mismatch" # Checks across datasets assert agesInInfectionMatrix == set( progression.keys()), "infection matrix and progression ages mismatch" assert agesInInfectionMatrix == {age for region in population.values() for age in region}, \ "infection matrix and population ages mismatch" disconnected_nodes = set(population.keys()) - set(graph.nodes()) if disconnected_nodes: log_issue( logger, f"These nodes have no contacts in the current network: {','.join(disconnected_nodes)}", IssueSeverity.MEDIUM, issues, ) state0: Dict[str, Dict[Tuple[str, str], float]] = {} for node in list(graph.nodes()): region = state0.setdefault(node, {}) for age, compartments in progression.items(): if node not in population: log_issue( logger, f"Node {node} is not in the population table, assuming population of 0 for all ages", IssueSeverity.MEDIUM, issues, ) pop = 0.0 else: pop = population[node][age] region[(age, SUSCEPTIBLE_STATE)] = pop for compartment in compartments: region[(age, compartment)] = 0 logger.info("Nodes: %s, Ages: %s, States: %s", len(state0), agesInInfectionMatrix, all_states) nop = NetworkOfPopulation( progression=progression, graph=graph, initialState=state0, mixingMatrix=mixingMatrix, movementMultipliers=movementMultipliers, infectiousStates=infectious_states, infectionProb=infection_prob, initialInfections=initial_infections, trials=trials, startDate=start_date, endDate=end_date, stochastic=stochastic_mode, ) return (nop, issues)
def test_basic_simulation(age_transitions, demographics, commute_moves, compartment_names, age_infection_matrix): age_to_trans = np.setUpParametersAges( loaders.readParametersAgeStructured(age_transitions)) population = loaders.readPopulationAgeStructured(demographics) graph = loaders.genGraphFromContactFile(commute_moves) states = np.setupInternalPopulations(graph, compartment_names, list(age_to_trans.keys()), population) result = np.basicSimulationInternalAgeStructure( rand=random.Random(1), graph=graph, numInfected=10, timeHorizon=200, genericInfection=0.1, ageInfectionMatrix=age_infection_matrix, diseaseProgressionProbs=age_to_trans, dictOfStates=states, ) expected = [ 0, 4.27, 5.959639, 7.495598979703686, 9.31292448442533, 11.550960285080539, 14.32076069530553, 17.750346856443436, 21.996595859880493, 27.253215606191763, 33.75958029276324, 41.81147883471944, 51.77434261313086, 64.09953959129047, 79.34443722167374, 98.19708895391234, 121.5065842268399, 150.32032333168627, 185.92974139403356, 229.92631464707316, 284.27004075722795, 351.3729944719362, 434.2010173334121, 536.3970957055917, 662.4304939975676, 817.7762023518806, 1009.1296689642635, 1244.6620222362512, 1534.3209088397605, 1890.1814875375615, 2326.8507704591293, 2861.926080186407, 3516.5045468063877, 4315.734994514654, 5289.3961276280115, 6472.475902589639, 7905.717433786824, 9636.08906655341, 11717.13446074682, 14209.16871256282, 17179.315999723975, 20701.439040274323, 24856.09052908778, 29730.707243910434, 35420.33217223618, 42029.128554931965, 49672.773024855385, 58481.44396269627, 68602.60309097261, 80202.27853153124, 93463.37789059673, 108579.96624726593, 125747.53880875603, 145150.90161910592, 166952.82966413427, 191287.47020551964, 218261.74745540041, 247965.3261465324, 280485.2367071461, 315916.38015373435, 354356.14564567205, 395872.69490304653, 440443.1364258809, 487868.5388265991, 537684.1860953799, 589091.5046792259, 640938.9095905811, 691769.9762481726, 739939.1843065555, 783772.9904113912, 821736.8679191938, 852567.1996643285, 875343.2246963251, 889499.9704796937, 894803.485637448, 891314.7525319818, 879359.4394061461, 859505.977536541, 832543.5198251844, 799448.8236543302, 761336.2708291251, 719393.6436135583, 674812.7866198674, 628726.0228195366, 582156.5631853839, 535986.5646189475, 490942.3244969345, 447593.5332414354, 406362.5717409869, 367540.01510870096, 331303.2145504978, 297735.6728639047, 266845.6949524783, 238583.40447859198, 212855.66475276518, 189538.74944848128, 168488.80761225612, 149550.28647197766, 132562.53827530914, 117364.86191178164, 103800.22980549172, 91717.93520231653, 80975.37134265121, 71439.12677319416, 62985.553271939556, 55500.93643588842, 48881.374979513224, 43032.45371205615, 37868.777124977256, 33313.415413891256, 29297.302334462536, 25758.61423633403, 22642.151601387908, 19898.738109895345, 17484.64737342592, 15361.063743396788, 13493.580801682876, 11851.739073032668, 10408.603009095552, 9140.37625068078, 8026.053473547509, 7047.106679898116, 6187.203546762776, 5431.955332005772, 4768.691828813375, 4186.260919647075, 3674.8503875840925, 3225.8297793633656, 2831.6102674926847, 2485.5206191429197, 2181.697540636489, 1914.9888235502897, 1680.8678687529227, 1475.3583061787265, 1294.9675597411838, 1136.6283280441162, 997.6470624276589, 875.6586246476982, 768.5863975872606, 674.6072044116167, 592.1204651473781, 519.721085459352, 456.1756310875024, 400.40139364220994, 351.44799986096695, 308.4812575877343, 270.76896818966605, 237.66846737351318, 208.61568486118352, 183.11553854396615, 160.73350093111426, 141.08819527652577, 123.8448960080496, 108.70982326635439, 95.42513472402354, 83.76452961376395, 73.52939023666997, 64.54539531517312, 56.65954754922215, 49.73756475976603, 43.66159017682611, 38.328182852791116, 33.64655394541582, 29.537018798984292, 25.929638426375362, 22.763027221103627, 19.98330656117008, 17.54318645356159, 15.401159551545785, 13.520793793496033, 11.870111594347737, 10.42104499749586, 9.148957491085977, 8.032224330272163, 7.051864205475331, 6.191215972979529, 5.435654933281273, 4.7723438175693484, 4.1900142350687, 3.678774853837169, 3.2299430438278187, 2.835897111408885, 2.4899466059017845, 2.186218487054244, 1.9195572129748524, 1.6854370455381997, 1.4798850786813753, 1.29941367791204, 1.1409611798590402, 1.001839841559402, 0.8796901527963475, 0.772440733292397, 0.6782731317724259, 0.5955909274692558, 0.5229926079757407, 0.45924776170425224, 0.40327617969564616, 0.35412951108933594, 0.3109751600700831, 0.2730821502880063, ] assert result == pytest.approx(expected)