def test_age_out_simulants(config, base_plugins):
start_population_size = 10000
num_days = 600
time_step = 100 # Days
config.update(
{
'population': {
'population_size': start_population_size,
'age_start': 4,
'age_end': 4,
'exit_age': 5,
},
'time': {
'step_size': time_step
}
},
layer='override')
components = [bp.BasePopulation()]
simulation = InteractiveContext(components=components,
configuration=config,
plugin_configuration=base_plugins)
time_start = simulation._clock.time
assert len(simulation.get_population()) == len(
simulation.get_population().age.unique())
simulation.run_for(duration=pd.Timedelta(days=num_days))
pop = simulation.get_population()
assert len(pop) == len(pop[~pop.tracked])
exit_after_300_days = pop.exit_time >= time_start + pd.Timedelta(300,
unit='D')
exit_before_400_days = pop.exit_time <= time_start + pd.Timedelta(400,
unit='D')
assert len(pop) == len(pop[exit_after_300_days & exit_before_400_days])
def test_lookup_table_interpolated_return_types(base_config):
year_start = base_config.time.start.year
year_end = base_config.time.end.year
data = build_table(lambda age, sex, year: year, year_start, year_end)
simulation = InteractiveContext(components=[TestPopulation()],
configuration=base_config)
manager = simulation._tables
table = (manager._build_table(data,
key_columns=['sex'],
parameter_columns=['age', 'year'],
value_columns=None)(
simulation.get_population().index))
# make sure a single value column is returned as a series
assert isinstance(table, pd.Series)
# now add a second value column to make sure the result is a df
data['value2'] = data.value
table = (manager._build_table(data,
key_columns=['sex'],
parameter_columns=['age', 'year'],
value_columns=None)(
simulation.get_population().index))
assert isinstance(table, pd.DataFrame)
def test_no_null_transition(base_config):
base_config.update({'population': {'population_size': 10000}})
a_state = State('a')
b_state = State('b')
start_state = State('start')
a_transition = Transition(
start_state,
a_state,
probability_func=lambda index: pd.Series(0.5, index=index))
b_transition = Transition(
start_state,
b_state,
probability_func=lambda index: pd.Series(0.5, index=index))
start_state.transition_set.allow_null_transition = False
start_state.transition_set.extend((a_transition, b_transition))
machine = Machine('state')
machine.states.extend([start_state, a_state, b_state])
simulation = InteractiveContext(
components=[machine, _population_fixture('state', 'start')],
configuration=base_config)
event_time = simulation._clock.time + simulation._clock.step_size
machine.transition(simulation.get_population().index, event_time)
a_count = (simulation.get_population().state == 'a').sum()
assert round(a_count / len(simulation.get_population()), 1) == 0.5
def test_BasePopulation(config, base_plugins, generate_population_mock):
num_days = 600
time_step = 100 # Days
sims = make_full_simulants()
start_population_size = len(sims)
generate_population_mock.return_value = sims.drop(columns=['tracked'])
base_pop = bp.BasePopulation()
components = [base_pop]
config.update(
{
'population': {
'population_size': start_population_size
},
'time': {
'step_size': time_step
}
},
layer='override')
simulation = InteractiveContext(components=components,
configuration=config,
plugin_configuration=base_plugins)
time_start = simulation._clock.time
pop_structure = simulation._data.load('population.structure')
pop_structure['location'] = simulation.configuration.input_data.location
uniform_pop = dt.assign_demographic_proportions(pop_structure)
assert base_pop.population_data.equals(uniform_pop)
age_params = {
'age_start': config.population.age_start,
'age_end': config.population.age_end
}
sub_pop = bp.BasePopulation.select_sub_population_data(
uniform_pop, time_start.year)
generate_population_mock.assert_called_once()
# Get a dictionary of the arguments used in the call
mock_args = generate_population_mock.call_args[1]
assert mock_args[
'creation_time'] == time_start - simulation._clock.step_size
assert mock_args['age_params'] == age_params
assert mock_args['population_data'].equals(sub_pop)
assert mock_args['randomness_streams'] == base_pop.randomness
pop = simulation.get_population()
for column in pop:
assert pop[column].equals(sims[column])
final_ages = pop.age + num_days / utilities.DAYS_PER_YEAR
simulation.run_for(duration=pd.Timedelta(days=num_days))
pop = simulation.get_population()
assert np.allclose(pop.age, final_ages, atol=0.5 /
utilities.DAYS_PER_YEAR) # Within a half of a day.
def test_transition():
done_state = State('done')
start_state = State('start')
start_state.add_transition(done_state)
machine = Machine('state', states=[start_state, done_state])
simulation = InteractiveContext(
components=[machine, _population_fixture('state', 'start')])
event_time = simulation._clock.time + simulation._clock.step_size
machine.transition(simulation.get_population().index, event_time)
assert np.all(simulation.get_population().state == 'done')
def test_dwell_time(assign_cause_mock, base_config, base_plugins, disease,
base_data):
time_step = 10
assign_cause_mock.side_effect = lambda population, *args: pd.DataFrame(
{'condition_state': 'healthy'}, index=population.index)
base_config.update(
{
'time': {
'step_size': time_step
},
'population': {
'population_size': 10
}
}, **metadata(__file__))
healthy_state = BaseDiseaseState('healthy')
data_function = base_data(0)
data_function['dwell_time'] = lambda _, __: pd.Timedelta(days=28)
data_function['disability_weight'] = lambda _, __: 0.0
event_state = DiseaseState('event', get_data_functions=data_function)
done_state = BaseDiseaseState('sick')
healthy_state.add_transition(event_state)
event_state.add_transition(done_state)
model = DiseaseModel(disease,
initial_state=healthy_state,
states=[healthy_state, event_state, done_state])
simulation = InteractiveContext(components=[TestPopulation(), model],
configuration=base_config,
plugin_configuration=base_plugins)
# Move everyone into the event state
simulation.step()
event_time = simulation._clock.time
assert np.all(simulation.get_population()[disease] == 'event')
simulation.step()
simulation.step()
# Not enough time has passed for people to move out of the event state, so they should all still be there
assert np.all(simulation.get_population()[disease] == 'event')
simulation.step()
# Now enough time has passed so people should transition away
assert np.all(simulation.get_population()[disease] == 'sick')
assert np.all(simulation.get_population().event_event_time ==
pd.to_datetime(event_time))
assert np.all(simulation.get_population().event_event_count == 1)
def test_Mortality(config, base_plugins):
num_days = 365
components = [TestPopulation(), Mortality()]
simulation = InteractiveContext(components=components,
configuration=config,
plugin_configuration=base_plugins,
setup=False)
df = pd.read_csv(config.path_to_mortality_file)
# to save time, only look at locatiosn existing on the test dataset.
mortality_rate_df = df[df['LAD.code'] == 'E08000032']
asfr_data = transform_rate_table(mortality_rate_df, 2011, 2012,
config.population.age_start,
config.population.age_end)
simulation._data.write("cause.all_causes.cause_specific_mortality_rate",
asfr_data)
simulation.setup()
simulation.run_for(duration=pd.Timedelta(days=num_days))
pop = simulation.get_population()
print('alive', len(pop[pop['alive'] == 'alive']))
print('dead', len(pop[pop['alive'] != 'alive']))
assert (np.all(pop.alive == 'alive') == False)
def test_incidence(base_config, base_plugins, disease):
year_start = base_config.time.start.year
year_end = base_config.time.end.year
time_step = pd.Timedelta(days=base_config.time.step_size)
healthy = BaseDiseaseState('healthy')
sick = BaseDiseaseState('sick')
key = f"sequela.acute_myocardial_infarction_first_2_days.incidence_rate"
transition = RateTransition(input_state=healthy,
output_state=sick,
get_data_functions={
'incidence_rate':
lambda _, builder: builder.data.load(key)
})
healthy.transition_set.append(transition)
model = DiseaseModel(disease,
initial_state=healthy,
states=[healthy, sick])
simulation = InteractiveContext(components=[TestPopulation(), model],
configuration=base_config,
plugin_configuration=base_plugins,
setup=False)
simulation._data.write(key, 0.7)
simulation.setup()
incidence_rate = simulation._values.get_value('sick.incidence_rate')
simulation.step()
assert np.allclose(from_yearly(0.7, time_step),
incidence_rate(simulation.get_population().index),
atol=0.00001)
def test_acmr():
sim = InteractiveContext(
'src/vivarium_conic_lsff/model_specifications/india.yaml')
sim.step() # make sure everything is setup
pop = sim.get_population()
age_groups = pd.cut(pop.age,
bins=[0, 7 / 365, 28 / 365, 1, 5],
right=False)
mr_pipeline = sim.get_value('mortality_rate')
acmr_orig = mr_pipeline.source(pop.index).sum(axis=1)
acmr_w_risk = mr_pipeline(
pop.index).sum(axis=1) * 365 # convert back to "per person-year"
# confirm that they are *not* identical at the individual level
assert not np.allclose(
acmr_orig, acmr_w_risk,
rtol=.05), 'expect acmr to be quite different for some individuals'
# but close at pop level
assert np.allclose(
acmr_orig.groupby(age_groups).median(),
acmr_w_risk.groupby(age_groups).median(),
rtol=.1
), 'expect acmr to be within 10% of original at population level'
def test_null_transition(base_config):
base_config.update({'population': {'population_size': 10000}})
a_state = State('a')
start_state = State('start')
start_state.add_transition(
a_state, probability_func=lambda agents: np.full(len(agents), 0.5))
start_state.allow_self_transitions()
machine = Machine('state', states=[start_state, a_state])
simulation = InteractiveContext(
components=[machine, _population_fixture('state', 'start')],
configuration=base_config)
event_time = simulation._clock.time + simulation._clock.step_size
machine.transition(simulation.get_population().index, event_time)
a_count = (simulation.get_population().state == 'a').sum()
assert round(a_count / len(simulation.get_population()), 1) == 0.5
def test_fertility_module(config, base_plugins):
start_population_size = config.population.population_size
num_days = 365 * 3
components = [TestPopulation(), FertilityAgeSpecificRates()]
simulation = InteractiveContext(components=components,
configuration=config,
plugin_configuration=base_plugins,
setup=False)
df = pd.read_csv(config.path_to_fertility_file)
# to save time, only look at locatiosn existing on the test dataset.
fertility_rate_df = df[(df['LAD.code'] == 'E08000032')]
asfr_data = transform_rate_table(fertility_rate_df, 2011, 2012, 10, 50,
[2])
# Mock Fertility Data
simulation._data.write("covariate.age_specific_fertility_rate.estimate",
asfr_data)
simulation.setup()
time_start = simulation._clock.time
assert 'last_birth_time' in simulation.get_population().columns, \
'expect Fertility module to update state table.'
assert 'parent_id' in simulation.get_population().columns, \
'expect Fertility module to update state table.'
simulation.run_for(duration=pd.Timedelta(days=num_days))
pop = simulation.get_population()
print(pop)
# No death in this model.
assert np.all(pop.alive == 'alive'), 'expect all simulants to be alive'
# TODO: Write a more rigorous test.
assert len(pop.age) > start_population_size, 'expect new simulants'
for i in range(start_population_size, len(pop)):
assert pop.loc[pop.iloc[i].parent_id].last_birth_time >= time_start, 'expect all children to have mothers who' \
' gave birth after the simulation starts.'
def test_lookup_table_scalar_from_single_value(base_config):
simulation = InteractiveContext(components=[TestPopulation()],
configuration=base_config)
manager = simulation._tables
table = (manager._build_table(1,
key_columns=None,
parameter_columns=None,
value_columns=['a'])(
simulation.get_population().index))
assert isinstance(table, pd.Series)
assert np.all(table == 1)
def test_risk_deletion(base_config, base_plugins, disease):
time_step = base_config.time.step_size
time_step = pd.Timedelta(days=time_step)
year_start = base_config.time.start.year
year_end = base_config.time.end.year
base_rate = 0.7
paf = 0.1
healthy = BaseDiseaseState('healthy')
sick = BaseDiseaseState('sick')
key = "sequela.acute_myocardial_infarction_first_2_days.incidence_rate"
transition = RateTransition(input_state=healthy,
output_state=sick,
get_data_functions={
'incidence_rate':
lambda _, builder: builder.data.load(key)
})
healthy.transition_set.append(transition)
model = DiseaseModel(disease,
initial_state=healthy,
states=[healthy, sick])
class PafModifier:
@property
def name(self):
return 'paf_modifier'
def setup(self, builder):
builder.value.register_value_modifier(
'sick.incidence_rate.paf',
modifier=simulation._tables.build_table(
build_table(paf, year_start, year_end),
key_columns=('sex', ),
parameter_columns=['age', 'year'],
value_columns=None))
simulation = InteractiveContext(
components=[TestPopulation(), model,
PafModifier()],
configuration=base_config,
plugin_configuration=base_plugins,
setup=False)
simulation._data.write(key, base_rate)
simulation.setup()
incidence_rate = simulation._values.get_value('sick.incidence_rate')
simulation.step()
expected_rate = base_rate * (1 - paf)
assert np.allclose(from_yearly(expected_rate, time_step),
incidence_rate(simulation.get_population().index),
atol=0.00001)
def test_interpolated_tables_without_uninterpolated_columns(base_config):
year_start = base_config.time.start.year
year_end = base_config.time.end.year
years = build_table(lambda age, sex, year: year, year_start, year_end)
del years['sex']
years = years.drop_duplicates()
base_config.update({
'population': {
'population_size': 10000
},
'interpolation': {
'order': 1
}
}) # the results we're checking later assume interp order 1
simulation = InteractiveContext(components=[TestPopulation()],
configuration=base_config)
manager = simulation._tables
years = manager.build_table(years,
key_columns=(),
parameter_columns=(
'year',
'age',
),
value_columns=None)
result_years = years(simulation.get_population().index)
fractional_year = simulation._clock.time.year
fractional_year += simulation._clock.time.timetuple().tm_yday / 365.25
assert np.allclose(result_years, fractional_year)
simulation._clock._time += pd.Timedelta(30.5 * 125, unit='D')
result_years = years(simulation.get_population().index)
fractional_year = simulation._clock.time.year
fractional_year += simulation._clock.time.timetuple().tm_yday / 365.25
assert np.allclose(result_years, fractional_year)
def test_side_effects():
class DoneState(State):
@property
def name(self):
return "test_done_state"
def setup(self, builder):
super().setup(builder)
self.population_view = builder.population.get_view(['count'])
def _transition_side_effect(self, index, event_time):
pop = self.population_view.get(index)
self.population_view.update(pop['count'] + 1)
done_state = DoneState('done')
start_state = State('start')
start_state.add_transition(done_state)
done_state.add_transition(start_state)
machine = Machine('state', states=[start_state, done_state])
simulation = InteractiveContext(components=[
machine,
_population_fixture('state', 'start'),
_population_fixture('count', 0)
])
event_time = simulation._clock.time + simulation._clock.step_size
machine.transition(simulation.get_population().index, event_time)
assert np.all(simulation.get_population()['count'] == 1)
machine.transition(simulation.get_population().index, event_time)
assert np.all(simulation.get_population()['count'] == 1)
machine.transition(simulation.get_population().index, event_time)
assert np.all(simulation.get_population()['count'] == 2)
def test_lookup_table_scalar_from_list(base_config):
simulation = InteractiveContext(components=[TestPopulation()],
configuration=base_config)
manager = simulation._tables
table = (manager._build_table(
(1, 2),
key_columns=None,
parameter_columns=None,
value_columns=['a', 'b'])(simulation.get_population().index))
assert isinstance(table, pd.DataFrame)
assert table.columns.values.tolist() == ['a', 'b']
assert np.all(table.a == 1)
assert np.all(table.b == 2)
def get_relative_risks(config: Path, input_draw: int, random_seed: int, age_group_id: int) -> pd.DataFrame:
sim = InteractiveContext(config, setup=False)
artifact_path = sim.configuration.input_data.artifact_path
artifact = Artifact(artifact_path)
age_bins = artifact.load(data_keys.POPULATION.AGE_BINS).reset_index().set_index('age_group_id')
age_start = age_bins.loc[age_group_id, 'age_start']
age_end = age_bins.loc[age_group_id, 'age_end']
year_start = 2019
year_end = 2020
sim.configuration.update({
'input_data': {
'input_draw_number': input_draw,
},
'randomness': {
'random_seed': random_seed,
},
'population': {
'age_start': age_start,
'age_end': age_end
}
})
sim.setup()
pop = sim.get_population()
gestational_ages = sim.get_value('short_gestation.exposure')(pop.index)
birth_weights = sim.get_value('low_birth_weight.exposure')(pop.index)
interpolators = artifact.load(data_keys.LBWSG.RELATIVE_RISK_INTERPOLATOR)
def calculate_rr_by_sex(sex: str) -> float:
sex_mask = pop['sex'] == sex
row_index = (sex, age_start, age_end, year_start, year_end, 'diarrheal_diseases', 'excess_mortality_rate')
interpolator = pickle.loads(bytes.fromhex(
interpolators.loc[row_index, f'draw_{input_draw}']
))
rrs = np.exp(interpolator(gestational_ages[sex_mask], birth_weights[sex_mask], grid=False))
return rrs
lbwsg_rrs = pd.DataFrame({'relative_risk': 1.0}, index=pop.index)
lbwsg_rrs['sex'] = pop['sex']
lbwsg_rrs.loc[lbwsg_rrs['sex'] == 'Female', 'relative_risk'] = calculate_rr_by_sex('Female')
lbwsg_rrs.loc[lbwsg_rrs['sex'] == 'Male', 'relative_risk'] = calculate_rr_by_sex('Male')
return lbwsg_rrs
def test_FertilityCrudeBirthRate(config, base_plugins):
pop_size = config.population.population_size
num_days = 100
components = [TestPopulation(), FertilityCrudeBirthRate()]
simulation = InteractiveContext(components=components,
configuration=config,
plugin_configuration=base_plugins,
setup=False)
simulation._data.write("covariate.live_births_by_sex.estimate", crude_birth_rate_data())
simulation.setup()
simulation.run_for(duration=pd.Timedelta(days=num_days))
pop = simulation.get_population()
assert np.all(pop.alive == 'alive')
assert len(pop.age) > pop_size
def test_interpolated_tables__exact_values_at_input_points(base_config):
year_start = base_config.time.start.year
year_end = base_config.time.end.year
years = build_table(lambda age, sex, year: year, year_start, year_end)
input_years = years.year_start.unique()
base_config.update({'population': {'population_size': 10000}})
simulation = InteractiveContext(components=[TestPopulation()],
configuration=base_config)
manager = simulation._tables
years = manager._build_table(years,
key_columns=['sex'],
parameter_columns=['age', 'year'],
value_columns=None)
for year in input_years:
simulation._clock._time = pd.Timestamp(year, 1, 1)
assert np.allclose(years(simulation.get_population().index),
simulation._clock.time.year + 1 / 365)
def test_FertilityDeterministic(config):
pop_size = config.population.population_size
annual_new_simulants = 1000
step_size = config.time.step_size
num_days = 100
config.update({
'fertility': {
'number_of_new_simulants_each_year': annual_new_simulants
}
}, **metadata(__file__))
components = [TestPopulation(), FertilityDeterministic()]
simulation = InteractiveContext(components=components, configuration=config)
num_steps = simulation.run_for(duration=pd.Timedelta(days=num_days))
pop = simulation.get_population()
assert num_steps == num_days // step_size
assert np.all(pop.alive == 'alive')
assert int(num_days * annual_new_simulants / utilities.DAYS_PER_YEAR) == len(pop.age) - pop_size
def test_mortality_rate(base_config, base_plugins, disease):
year_start = base_config.time.start.year
year_end = base_config.time.end.year
time_step = pd.Timedelta(days=base_config.time.step_size)
healthy = BaseDiseaseState('healthy')
mort_get_data_funcs = {
'dwell_time':
lambda _, __: pd.Timedelta(days=0),
'disability_weight':
lambda _, __: 0.0,
'prevalence':
lambda _, __: build_table(0.000001, year_start - 1, year_end,
['age', 'year', 'sex', 'value']),
'excess_mortality_rate':
lambda _, __: build_table(0.7, year_start - 1, year_end),
}
mortality_state = DiseaseState('sick',
get_data_functions=mort_get_data_funcs)
healthy.add_transition(mortality_state)
model = DiseaseModel(disease,
initial_state=healthy,
states=[healthy, mortality_state])
simulation = InteractiveContext(
components=[TestPopulation(), model,
Mortality()],
configuration=base_config,
plugin_configuration=base_plugins)
mortality_rate = simulation._values.get_value('mortality_rate')
simulation.step()
# Folks instantly transition to sick so now our mortality rate should be much higher
assert np.allclose(
from_yearly(0.7, time_step),
mortality_rate(simulation.get_population().index)['sick'])
def test_emigration(config, base_plugins):
start_population_size = config.population.population_size
num_days = 365 * 10
components = [TestPopulation(), Emigration()]
simulation = InteractiveContext(components=components,
configuration=config,
plugin_configuration=base_plugins,
setup=False)
# setup emigration rates
df_emigration = pd.read_csv(config.path_to_emigration_file)
df_total_population = pd.read_csv(config.path_to_total_population_file)
df_emigration = df_emigration[(df_emigration['LAD.code'] == 'E08000032') |
(df_emigration['LAD.code'] == 'E08000032')]
df_total_population = df_total_population[
(df_total_population['LAD'] == 'E08000032') |
(df_total_population['LAD'] == 'E08000032')]
asfr_data_emigration = compute_migration_rates(df_emigration,
df_total_population,
2011,
2012,
config.population.age_start,
config.population.age_end,
aggregate_over=75)
# Mock emigration Data
simulation._data.write("covariate.age_specific_migration_rate.estimate",
asfr_data_emigration)
simulation.setup()
simulation.run_for(duration=pd.Timedelta(days=num_days))
pop = simulation.get_population()
print('emigrated', len(pop[pop['alive'] == 'emigrated']))
print('remaining population', len(pop[pop['emigrated'] == 'no_emigrated']))
assert (np.all(pop.alive == 'alive') == False)
assert len(pop[pop['emigrated'] == 'Yes']) > 0, 'expect migration'
def test_Immigration(config, base_plugins):
num_days = 10
components = [TestPopulation(), Immigration()]
simulation = InteractiveContext(components=components,
configuration=config,
plugin_configuration=base_plugins,
setup=False)
df_total_population = pd.read_csv(config.path_to_total_population_file)
df_total_population = df_total_population[
(df_total_population['LAD'] == 'E08000032')]
# setup immigration rates
df_immigration = pd.read_csv(config.path_to_immigration_file)
df_immigration = df_immigration[
(df_immigration['LAD.code'] == 'E08000032')]
asfr_data_immigration = compute_migration_rates(df_immigration, df_total_population,
2011,
2012,
config.population.age_start,
config.population.age_end,
normalize=False
)
# setup immigration rates
df_immigration_MSOA = pd.read_csv(config.path_to_immigration_MSOA)
# read total immigrants from the file
total_immigrants = int(df_immigration[df_immigration.columns[4:]].sum().sum())
simulation._data.write("cause.all_causes.cause_specific_immigration_rate", asfr_data_immigration)
simulation._data.write("cause.all_causes.cause_specific_total_immigrants_per_year", total_immigrants)
simulation._data.write("cause.all_causes.immigration_to_MSOA", df_immigration_MSOA)
simulation.setup()
simulation.run_for(duration=pd.Timedelta(days=num_days))
pop = simulation.get_population()
assert (len(pop["entrance_time"].value_counts()) > 1)
print (pop)
def test_internal_outmigration(config, base_plugins):
num_days = 365 * 5
components = [TestPopulation(), InternalMigration()]
simulation = InteractiveContext(components=components,
configuration=config,
plugin_configuration=base_plugins,
setup=False)
df = pd.read_csv(config.path_to_internal_outmigration_file)
# to save time, only look at locations existing on the test dataset.
df_internal_outmigration = df[df['LAD.code'].isin([
'E08000032', 'E08000033', 'E08000034', 'E06000024', 'E08000035',
'E07000163'
])]
asfr_data = transform_rate_table(df_internal_outmigration, 2011, 2012,
config.population.age_start,
config.population.age_end)
simulation._data.write("cause.age_specific_internal_outmigration_rate",
asfr_data)
# Read MSOA ---> LAD
msoa_lad_df = pd.read_csv(config.path_msoa_to_lad)
# Read OD matrix, only destinations
OD_matrix_dest = pd.read_csv(config.path_to_OD_matrix_index_file,
index_col=0)
OD_matrix_with_LAD = OD_matrix_dest.merge(
msoa_lad_df[["MSOA11CD", "LAD16CD"]],
left_index=True,
right_on="MSOA11CD")
OD_matrix_with_LAD.index = OD_matrix_with_LAD["indices"]
# Create indices for MSOA and LAD
MSOA_location_index = OD_matrix_with_LAD["MSOA11CD"].to_dict()
LAD_location_index = OD_matrix_with_LAD["LAD16CD"].to_dict()
# Now, read the whole matrix (if it passes the first check)
simulation._data.write("internal_migration.MSOA_index",
MSOA_location_index)
simulation._data.write("internal_migration.LAD_index", LAD_location_index)
simulation._data.write("internal_migration.MSOA_LAD_indices",
OD_matrix_with_LAD)
simulation._data.write("internal_migration.path_to_OD_matrices",
config.path_to_OD_matrices)
simulation.setup()
simulation.run_for(duration=pd.Timedelta(days=num_days))
pop = simulation.get_population()
print('internal outmigration',
len(pop[pop['internal_outmigration'] == 'Yes']))
print('remaining population',
len(pop[pop['internal_outmigration'] == 'No']))
assert (np.all(pop.internal_outmigration == 'Yes') == False)
assert len(pop[pop['last_outmigration_time'] != 'NaT']
) > 0, 'time of out migration gets saved.'
assert len(pop[pop['previous_MSOA_locations'] != '']
) > 0, 'previous location of the migrant gets saved.'
def test_pipeline(config, base_plugins):
start_population_size = config.population.population_size
num_days = 365 * 2
components = [
TestPopulation(),
FertilityAgeSpecificRates(),
Mortality(),
Emigration(),
Immigration()
]
simulation = InteractiveContext(components=components,
configuration=config,
plugin_configuration=base_plugins,
setup=False)
# setup mortality rates
df = pd.read_csv(config.path_to_mortality_file)
mortality_rate_df = df[(df['LAD.code'] == 'E08000032')]
asfr_data = transform_rate_table(mortality_rate_df, 2011, 2012,
config.population.age_start,
config.population.age_end)
simulation._data.write("cause.all_causes.cause_specific_mortality_rate",
asfr_data)
# setup fertility rates
df_fertility = pd.read_csv(config.path_to_fertility_file)
fertility_rate_df = df_fertility[(df_fertility['LAD.code'] == 'E08000032')]
asfr_data_fertility = transform_rate_table(fertility_rate_df, 2011, 2012,
10, 50, [2])
simulation._data.write("covariate.age_specific_fertility_rate.estimate",
asfr_data_fertility)
# setup emigration rates
df_emigration = pd.read_csv(config.path_to_emigration_file)
df_total_population = pd.read_csv(config.path_to_total_population_file)
df_emigration = df_emigration[(df_emigration['LAD.code'] == 'E08000032')]
df_total_population = df_total_population[(
df_total_population['LAD'] == 'E08000032')]
asfr_data_emigration = compute_migration_rates(df_emigration,
df_total_population, 2011,
2012,
config.population.age_start,
config.population.age_end)
simulation._data.write("covariate.age_specific_migration_rate.estimate",
asfr_data_emigration)
# setup immigration rates
df_immigration = pd.read_csv(config.path_to_immigration_file)
df_immigration = df_immigration[(
df_immigration['LAD.code'] == 'E08000032')]
asfr_data_immigration = compute_migration_rates(
df_immigration,
df_total_population,
2011,
2012,
config.population.age_start,
config.population.age_end,
normalize=False)
# read total immigrants from the file
total_immigrants = int(
df_immigration[df_immigration.columns[4:]].sum().sum())
simulation._data.write("cause.all_causes.cause_specific_immigration_rate",
asfr_data_immigration)
simulation._data.write(
"cause.all_causes.cause_specific_total_immigrants_per_year",
total_immigrants)
df_immigration_MSOA = pd.read_csv(config.path_to_immigration_MSOA)
simulation._data.write("cause.all_causes.immigration_to_MSOA",
df_immigration_MSOA)
simulation.setup()
time_start = simulation._clock.time
assert 'last_birth_time' in simulation.get_population().columns, \
'expect Fertility module to update state table.'
assert 'parent_id' in simulation.get_population().columns, \
'expect Fertility module to update state table.'
simulation.run_for(duration=pd.Timedelta(days=num_days))
pop = simulation.get_population()
print('alive', len(pop[pop['alive'] == 'alive']))
print('dead', len(pop[pop['alive'] == 'dead']))
print('emigrated', len(pop[pop['alive'] == 'emigrated']))
assert (np.all(pop.alive == 'alive') == False)
assert len(pop[pop['emigrated'] == 'Yes']) > 0, 'expect migration'
assert len(pop.age) > start_population_size, 'expect new simulants'
for i in range(start_population_size, len(pop)):
# skip immigrated population
if pop.loc[i].immigrated == 'Yes':
continue
assert pop.loc[pop.loc[i].parent_id].last_birth_time >= time_start, 'expect all children to have mothers who' \
' gave birth after the simulation starts.'
def RunPipeline(config, start_population_size):
""" Run the daedalus Microsimulation pipeline
Parameters
----------
config : ConfigTree
Config file to run the pipeline
start_population_size: int
Size of the starting population
Returns:
--------
A dataframe with the resulting simulation
"""
# Set up the components using the config.
config.update({'population': {
'population_size': start_population_size,
}},
source=str(Path(__file__).resolve()))
num_years = config.time.num_years
components = [eval(x) for x in config.components]
simulation = InteractiveContext(components=components,
configuration=config,
plugin_configuration=utils.base_plugins(),
setup=False)
if 'InternalMigration()' in config.components:
# setup internal migration matrices
OD_matrices = InternalMigrationMatrix(configuration=config)
OD_matrices.set_matrix_tables()
simulation._data.write("internal_migration.MSOA_index",
OD_matrices.MSOA_location_index)
simulation._data.write("internal_migration.LAD_index",
OD_matrices.LAD_location_index)
simulation._data.write("internal_migration.MSOA_LAD_indices",
OD_matrices.df_OD_matrix_with_LAD)
simulation._data.write("internal_migration.path_to_OD_matrices",
config.path_to_OD_matrices)
# setup internal migraionts rates
asfr_int_migration = InternalMigrationRateTable(configuration=config)
asfr_int_migration.set_rate_table()
simulation._data.write("cause.age_specific_internal_outmigration_rate",
asfr_int_migration.rate_table)
if 'Mortality()' in config.components:
# setup mortality rates
asfr_mortality = MortalityRateTable(configuration=config)
asfr_mortality.set_rate_table()
simulation._data.write(
"cause.all_causes.cause_specific_mortality_rate",
asfr_mortality.rate_table)
if 'FertilityAgeSpecificRates()' in config.components:
# setup fertility rates
asfr_fertility = FertilityRateTable(configuration=config)
asfr_fertility.set_rate_table()
simulation._data.write(
"covariate.age_specific_fertility_rate.estimate",
asfr_fertility.rate_table)
if 'Emigration()' in config.components:
# setup emigration rates
asfr_emigration = EmigrationRateTable(configuration=config)
asfr_emigration.set_rate_table()
simulation._data.write(
"covariate.age_specific_migration_rate.estimate",
asfr_emigration.rate_table)
if 'Immigration()' in config.components:
# setup immigration rates
asfr_immigration = ImmigrationRateTable(configuration=config)
asfr_immigration.set_rate_table()
asfr_immigration.set_total_immigrants()
simulation._data.write("cause.all_causes.immigration_to_MSOA",
pd.read_csv(config.path_to_immigration_MSOA))
simulation._data.write(
"cause.all_causes.cause_specific_immigration_rate",
asfr_immigration.rate_table)
simulation._data.write(
"cause.all_causes.cause_specific_total_immigrants_per_year",
asfr_immigration.total_immigrants)
print('Start simulation setup')
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
simulation.setup()
print('Start running simulation')
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
for year in range(1, num_years + 1):
simulation.run_for(duration=pd.Timedelta(days=365.25))
print('Finished running simulation for year:', year)
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
pop = simulation.get_population()
# assign age brackets to the individuals
pop = utils.get_age_bucket(pop)
# save the output file to csv
year_output_dir = os.path.join(
os.path.join(config.output_dir, config.location,
'year_' + str(year)))
os.makedirs(year_output_dir, exist_ok=True)
output_data_filename = 'ssm_' + config.location + '_MSOA11_ppp_2011_simulation_year_' + str(
year) + '.csv'
pop.to_csv(os.path.join(year_output_dir, output_data_filename))
print()
print('In year: ', config.time.start.year + year)
# print some summary stats on the simulation
print('alive', len(pop[pop['alive'] == 'alive']))
if 'Mortality()' in config.components:
print('dead', len(pop[pop['alive'] == 'dead']))
if 'Emigration()' in config.components:
print('emigrated', len(pop[pop['alive'] == 'emigrated']))
if 'InternalMigration()' in config.components:
print('internal migration',
len(pop[pop['internal_outmigration'] != '']))
if 'FertilityAgeSpecificRates()' in config.components:
print('New children', len(pop[pop['parent_id'] != -1]))
if 'Immigration()' in config.components:
print('Immigrants',
len(pop[pop['immigrated'].astype(str) == 'Yes']))
return pop
def test_dwell_time_with_mortality(base_config, base_plugins, disease):
year_start = base_config.time.start.year
year_end = base_config.time.end.year
time_step = 10
pop_size = 100
base_config.update(
{
'time': {
'step_size': time_step
},
'population': {
'population_size': pop_size
}
}, **metadata(__file__))
healthy_state = BaseDiseaseState('healthy')
mort_get_data_funcs = {
'dwell_time':
lambda _, __: pd.Timedelta(days=14),
'excess_mortality_rate':
lambda _, __: build_table(0.7, year_start - 1, year_end),
'disability_weight':
lambda _, __: 0.0
}
mortality_state = DiseaseState('event',
get_data_functions=mort_get_data_funcs)
done_state = BaseDiseaseState('sick')
healthy_state.add_transition(mortality_state)
mortality_state.add_transition(done_state)
model = DiseaseModel(disease,
initial_state=healthy_state,
states=[healthy_state, mortality_state, done_state])
mortality = Mortality()
simulation = InteractiveContext(
components=[TestPopulation(), model, mortality],
configuration=base_config,
plugin_configuration=base_plugins)
# Move everyone into the event state
simulation.step()
assert np.all(simulation.get_population()[disease] == 'event')
simulation.step()
# Not enough time has passed for people to move out of the event state, so they should all still be there
assert np.all(simulation.get_population()[disease] == 'event')
simulation.step()
# Make sure some people have died and remained in event state
assert (simulation.get_population()['alive'] == 'alive').sum() < pop_size
assert ((simulation.get_population()['alive'] == 'dead').sum() == (
simulation.get_population()[disease] == 'event').sum())
# enough time has passed so living people should transition away to sick
assert ((simulation.get_population()['alive'] == 'alive').sum() == (
simulation.get_population()[disease] == 'sick').sum())
def test_interpolated_tables(base_config):
year_start = base_config.time.start.year
year_end = base_config.time.end.year
years = build_table(lambda age, sex, year: year, year_start, year_end)
ages = build_table(lambda age, sex, year: age, year_start, year_end)
one_d_age = ages.copy()
del one_d_age['year']
one_d_age = one_d_age.drop_duplicates()
base_config.update({
'population': {
'population_size': 10000
},
'interpolation': {
'order': 1
}
}) # the results we're checking later assume interp order 1
simulation = InteractiveContext(components=[TestPopulation()],
configuration=base_config)
manager = simulation._tables
years = manager.build_table(years,
key_columns=('sex', ),
parameter_columns=(
'age',
'year',
),
value_columns=None)
ages = manager.build_table(ages,
key_columns=('sex', ),
parameter_columns=(
'age',
'year',
),
value_columns=None)
one_d_age = manager.build_table(one_d_age,
key_columns=('sex', ),
parameter_columns=('age', ),
value_columns=None)
pop = simulation.get_population(untracked=True)
result_years = years(pop.index)
result_ages = ages(pop.index)
result_ages_1d = one_d_age(pop.index)
fractional_year = simulation._clock.time.year
fractional_year += simulation._clock.time.timetuple().tm_yday / 365.25
assert np.allclose(result_years, fractional_year)
assert np.allclose(result_ages, pop.age)
assert np.allclose(result_ages_1d, pop.age)
simulation._clock._time += pd.Timedelta(30.5 * 125, unit='D')
simulation._population._population.age += 125 / 12
result_years = years(pop.index)
result_ages = ages(pop.index)
result_ages_1d = one_d_age(pop.index)
fractional_year = simulation._clock.time.year
fractional_year += simulation._clock.time.timetuple().tm_yday / 365.25
assert np.allclose(result_years, fractional_year)
assert np.allclose(result_ages, pop.age)
assert np.allclose(result_ages_1d, pop.age)
