def continuous_risk():
year_start = 1990
year_end = 2010
risk = 'test_risk'
risk_data = dict()
exposure_mean = make_test_data_table([130])
exposure_sd = make_test_data_table([15])
affected_causes = ["test_cause_1", "test_cause_2"]
rr_data = []
paf_data = []
for cause in affected_causes:
rr_data.append(
build_table(
[1.01, cause],
year_start,
year_end,
['age', 'sex', 'year', 'value', 'cause'],
).melt(id_vars=('age_start', 'age_end', 'year_start', 'year_end',
'sex', 'cause'),
var_name='parameter',
value_name='value'))
paf_data.append(
build_table([1, cause], year_start, year_end,
['age', 'sex', 'year', 'value', 'cause']))
rr_data = pd.concat(rr_data)
paf_data = pd.concat(paf_data)
paf_data['affected_measure'] = 'incidence_rate'
rr_data['affected_measure'] = 'incidence_rate'
risk_data['exposure'] = exposure_mean
risk_data['exposure_standard_deviation'] = exposure_sd
risk_data['relative_risk'] = rr_data
risk_data['population_attributable_fraction'] = paf_data
risk_data['affected_causes'] = affected_causes
risk_data['affected_risk_factors'] = []
tmred = {
"distribution": 'uniform',
"min": 110.0,
"max": 115.0,
"inverted": False,
}
exposure_parameters = {
"scale": 10.0,
"max_rr": 200.0,
"max_val": 300.0,
"min_val": 50.0,
}
tmrel = 0.5 * (tmred["max"] + tmred["min"])
risk_data['tmred'] = tmred
risk_data['tmrel'] = tmrel
risk_data['exposure_parameters'] = exposure_parameters
risk_data['distribution'] = 'normal'
return Risk(f"risk_factor.{risk}"), risk_data
def coverage_gap():
year_start = 1990
year_end = 2010
cg = 'test_coverage_gap'
cg_data = dict()
cg_exposed = 0.6
cg_exposure_data = build_table(
[cg_exposed, 1 - cg_exposed], year_start, year_end,
['age', 'year', 'sex', 'cat1', 'cat2']).melt(id_vars=(
'age_start',
'age_end',
'year_start',
'year_end',
'sex',
),
var_name='parameter',
value_name='value')
rr = 2
rr_data = build_table([rr, 1], year_start, year_end,
['age', 'year', 'sex', 'cat1', 'cat2']).melt(
id_vars=('age_start', 'age_end', 'year_start',
'year_end', 'sex'),
var_name='parameter',
value_name='value')
# paf is (sum(exposure(category)*rr(category) -1 )/ (sum(exposure(category)* rr(category)
paf = (rr * cg_exposed + (1 - cg_exposed) - 1) / (rr * cg_exposed +
(1 - cg_exposed))
paf_data = build_table(
paf, year_start, year_end,
['age', 'year', 'sex', 'population_attributable_fraction']).melt(
id_vars=('age_start', 'age_end', 'year_start', 'year_end', 'sex'),
var_name='population_attributable_fraction',
value_name='value')
paf_data['risk_factor'] = 'test_risk'
paf_data['affected_measure'] = 'exposure_parameters'
rr_data['affected_measure'] = 'exposure_parameters'
cg_data['exposure'] = cg_exposure_data
rr_data['risk_factor'] = 'test_risk'
cg_data['relative_risk'] = rr_data
cg_data['population_attributable_fraction'] = paf_data
cg_data['affected_causes'] = []
cg_data['affected_risk_factors'] = ['test_risk']
cg_data['distribution'] = 'dichotomous'
return Risk(f'coverage_gap.{cg}'), cg_data
def test_lookup_table_interpolated_return_types(base_config):
year_start = base_config.time.start.year
year_end = base_config.time.end.year
data = build_table(lambda age, sex, year: year, year_start, year_end)
simulation = InteractiveContext(components=[TestPopulation()],
configuration=base_config)
manager = simulation._tables
table = (manager._build_table(data,
key_columns=['sex'],
parameter_columns=['age', 'year'],
value_columns=None)(
simulation.get_population().index))
# make sure a single value column is returned as a series
assert isinstance(table, pd.Series)
# now add a second value column to make sure the result is a df
data['value2'] = data.value
table = (manager._build_table(data,
key_columns=['sex'],
parameter_columns=['age', 'year'],
value_columns=None)(
simulation.get_population().index))
assert isinstance(table, pd.DataFrame)
def make_test_data_table(values: List, parameter='cat') -> pd.DataFrame:
year_start = 1990 # same as the base config
year_end = 2010
if len(values) == 1:
df = build_table(values[0], year_start, year_end,
('age', 'year', 'sex', 'value'))
else:
cats = [f'{parameter}{i+1}' for i in range(len(values))
] if parameter == 'cat' else parameter
df = []
for cat, value in zip(cats, values):
df.append(
build_table([cat, value], year_start, year_end,
('age', 'year', 'sex', 'parameter', 'value')))
df = pd.concat(df)
return df
def setup(self, builder):
builder.value.register_value_modifier(
'sick.incidence_rate.paf',
modifier=simulation._tables.build_table(
build_table(paf, year_start, year_end),
key_columns=('sex', ),
parameter_columns=['age', 'year'],
value_columns=None))
def test_mortality_rate(base_config, base_plugins, disease):
year_start = base_config.time.start.year
year_end = base_config.time.end.year
time_step = pd.Timedelta(days=base_config.time.step_size)
healthy = BaseDiseaseState('healthy')
mort_get_data_funcs = {
'dwell_time':
lambda _, __: pd.Timedelta(days=0),
'disability_weight':
lambda _, __: 0.0,
'prevalence':
lambda _, __: build_table(0.000001, year_start - 1, year_end,
['age', 'year', 'sex', 'value']),
'excess_mortality_rate':
lambda _, __: build_table(0.7, year_start - 1, year_end),
}
mortality_state = DiseaseState('sick',
get_data_functions=mort_get_data_funcs)
healthy.add_transition(mortality_state)
model = DiseaseModel(disease,
initial_state=healthy,
states=[healthy, mortality_state])
simulation = InteractiveContext(
components=[TestPopulation(), model,
Mortality()],
configuration=base_config,
plugin_configuration=base_plugins)
mortality_rate = simulation._values.get_value('mortality_rate')
simulation.step()
# Folks instantly transition to sick so now our mortality rate should be much higher
assert np.allclose(
from_yearly(0.7, time_step),
mortality_rate(simulation.get_population().index)['sick'])
def dichotomous_risk():
year_start = 1990
year_end = 2010
risk = 'test_risk'
risk_data = dict()
exposure_data = build_table(0.5, year_start, year_end,
['age', 'year', 'sex', 'cat1', 'cat2']).melt(
id_vars=('age_start', 'age_end',
'year_start', 'year_end', 'sex'),
var_name='parameter',
value_name='value')
affected_causes = ["test_cause_1", "test_cause_2"]
rr_data = []
paf_data = []
for cause in affected_causes:
rr_data.append(
build_table([1.01, 1, cause], year_start, year_end,
['age', 'year', 'sex', 'cat1', 'cat2', 'cause']).melt(
id_vars=('age_start', 'age_end', 'year_start',
'year_end', 'sex', 'cause'),
var_name='parameter',
value_name='value'))
paf_data.append(
build_table([1, cause], year_start, year_end,
['age', 'sex', 'year', 'value', 'cause']))
rr_data = pd.concat(rr_data)
paf_data = pd.concat(paf_data)
paf_data['affected_measure'] = 'incidence_rate'
rr_data['affected_measure'] = 'incidence_rate'
risk_data['exposure'] = exposure_data
risk_data['relative_risk'] = rr_data
risk_data['population_attributable_fraction'] = paf_data
risk_data['affected_causes'] = affected_causes
risk_data['affected_risk_factors'] = []
incidence_rate = build_table(0.01, year_start, year_end)
risk_data['incidence_rate'] = incidence_rate
risk_data['distribution'] = 'dichotomous'
return Risk(f'risk_factor.{risk}'), risk_data
def load(self, entity_key):
if entity_key in self.mocks:
return self.mocks[entity_key]
entity_type, *_, entity_measure = entity_key.split('.')
assert entity_type in self.mocks
assert entity_measure in self.mocks[entity_type]
value = self.mocks[entity_type][entity_measure]
if callable(value):
value = value(entity_key)
elif not isinstance(value, (pd.DataFrame, pd.Series)):
value = build_table(value, 1990, 2018)
return value
def test_subset_rows():
values = [
lambda *args, **kwargs: random.choice(['red', 'blue']),
lambda *args, **kwargs: random.choice([1, 2, 3])
]
data = build_table(values,
1990,
2010,
columns=('age', 'year', 'sex', 'color', 'number'))
filtered_data = _subset_rows(data, color='red', number=3)
assert filtered_data.equals(data[(data.color == 'red')
& (data.number == 3)])
filtered_data = _subset_rows(data, color='red', number=[2, 3])
assert filtered_data.equals(
data[(data.color == 'red')
& ((data.number == 2) | (data.number == 3))])
def test_subset_columns():
values = [0, 'Kenya', 'red', 100]
data = build_table(values,
1990,
2010,
columns=('age', 'year', 'sex', 'draw', 'location',
'color', 'value'))
filtered_data = _subset_columns(data)
assert filtered_data.equals(data[[
'age_start', 'age_end', 'year_start', 'year_end', 'sex', 'color',
'value'
]])
filtered_data = _subset_columns(data, color='red')
assert filtered_data.equals(data[[
'age_start', 'age_end', 'year_start', 'year_end', 'sex', 'value'
]])
def test_interpolated_tables__exact_values_at_input_points(base_config):
year_start = base_config.time.start.year
year_end = base_config.time.end.year
years = build_table(lambda age, sex, year: year, year_start, year_end)
input_years = years.year_start.unique()
base_config.update({'population': {'population_size': 10000}})
simulation = InteractiveContext(components=[TestPopulation()],
configuration=base_config)
manager = simulation._tables
years = manager._build_table(years,
key_columns=['sex'],
parameter_columns=['age', 'year'],
value_columns=None)
for year in input_years:
simulation._clock._time = pd.Timestamp(year, 1, 1)
assert np.allclose(years(simulation.get_population().index),
simulation._clock.time.year + 1 / 365)
def test_write_data_frame(hdf_file_path):
key = hdf.EntityKey('cause.test.prevalence')
data = build_table(
[lambda *args, **kwargs: random.choice([0, 1]), "Kenya", 1],
2005,
2010,
columns=('age', 'year', 'sex', 'draw', 'location', 'value'))
non_val_columns = data.columns.difference({'value'})
data = data.set_index(list(non_val_columns))
hdf._write_pandas_data(hdf_file_path, key, data)
written_data = pd.read_hdf(hdf_file_path, key.path)
assert written_data.equals(data)
filter_terms = ['draw == 0']
written_data = pd.read_hdf(hdf_file_path, key.path, where=filter_terms)
assert written_data.equals(data.xs(0, level='draw', drop_level=False))
def test_fertility_module(base_config, base_plugins):
start_population_size = 1000
num_days = 1000
time_step = 10 # Days
base_config.update({
'population': {
'population_size': start_population_size,
'age_start': 0,
'age_end': 125},
'time': {'step_size': time_step}
}, layer='override')
components = [TestPopulation(), FertilityAgeSpecificRates()]
simulation = simulation = InteractiveContext(components=components,
configuration=base_config,
plugin_configuration=base_plugins,
setup=False)
asfr_data = build_table(0.05, 1990, 2017).rename(columns={'value': 'mean_value'})
simulation._data.write("covariate.age_specific_fertility_rate.estimate", asfr_data)
simulation.setup()
time_start = simulation._clock.time
assert 'last_birth_time' in simulation.get_population().columns,\
'expect Fertility module to update state table.'
assert 'parent_id' in simulation.get_population().columns, \
'expect Fertility module to update state table.'
simulation.run_for(duration=pd.Timedelta(days=num_days))
pop = simulation.get_population()
# No death in this model.
assert np.all(pop.alive == 'alive'), 'expect all simulants to be alive'
# TODO: Write a more rigorous test.
assert len(pop.age) > start_population_size, 'expect new simulants'
for i in range(start_population_size, len(pop)):
assert pop.loc[pop.iloc[i].parent_id].last_birth_time >= time_start, 'expect all children to have mothers who' \
' gave birth after the simulation starts.'
def test_interpolated_tables_without_uninterpolated_columns(base_config):
year_start = base_config.time.start.year
year_end = base_config.time.end.year
years = build_table(lambda age, sex, year: year, year_start, year_end)
del years['sex']
years = years.drop_duplicates()
base_config.update({
'population': {
'population_size': 10000
},
'interpolation': {
'order': 1
}
}) # the results we're checking later assume interp order 1
simulation = InteractiveContext(components=[TestPopulation()],
configuration=base_config)
manager = simulation._tables
years = manager.build_table(years,
key_columns=(),
parameter_columns=(
'year',
'age',
),
value_columns=None)
result_years = years(simulation.get_population().index)
fractional_year = simulation._clock.time.year
fractional_year += simulation._clock.time.timetuple().tm_yday / 365.25
assert np.allclose(result_years, fractional_year)
simulation._clock._time += pd.Timedelta(30.5 * 125, unit='D')
result_years = years(simulation.get_population().index)
fractional_year = simulation._clock.time.year
fractional_year += simulation._clock.time.timetuple().tm_yday / 365.25
assert np.allclose(result_years, fractional_year)
MOCKERS = {
'cause': {
'prevalence': 0,
'cause_specific_mortality_rate': 0,
'excess_mortality_rate': 0,
'remission_rate': 0,
'incidence_rate': 0.001,
'disability_weight': pd.DataFrame({'value': [0]}),
'restrictions': lambda *args, **kwargs: {'yld_only': False}
},
'risk_factor': {
'distribution': lambda *args, **kwargs: 'ensemble',
'exposure': 120,
'exposure_standard_deviation': 15,
'relative_risk': build_table([1.5, "continuous", "test_cause", "incidence_rate"], 1990, 2017,
("age", "sex", "year", "value", "parameter", "cause", "affected_measure")),
'population_attributable_fraction': build_table([1, "test_cause_1", "incidence_rate"], 1990, 2017,
("age", "sex", "year", "value", "cause", "affected_measure")),
'tmred': lambda *args, **kwargs: {
"distribution": "uniform",
"min": 80,
"max": 100,
"inverted": False,
},
'exposure_parameters': lambda *args, **kwargs: {
'scale': 1,
'max_rr': 10,
'max_val': 200,
'min_val': 0,
},
'ensemble_weights': lambda *args, **kwargs: pd.DataFrame({'norm': 1}, index=[0])
def test_dwell_time_with_mortality(base_config, base_plugins, disease):
year_start = base_config.time.start.year
year_end = base_config.time.end.year
time_step = 10
pop_size = 100
base_config.update(
{
'time': {
'step_size': time_step
},
'population': {
'population_size': pop_size
}
}, **metadata(__file__))
healthy_state = BaseDiseaseState('healthy')
mort_get_data_funcs = {
'dwell_time':
lambda _, __: pd.Timedelta(days=14),
'excess_mortality_rate':
lambda _, __: build_table(0.7, year_start - 1, year_end),
'disability_weight':
lambda _, __: 0.0
}
mortality_state = DiseaseState('event',
get_data_functions=mort_get_data_funcs)
done_state = BaseDiseaseState('sick')
healthy_state.add_transition(mortality_state)
mortality_state.add_transition(done_state)
model = DiseaseModel(disease,
initial_state=healthy_state,
states=[healthy_state, mortality_state, done_state])
mortality = Mortality()
simulation = InteractiveContext(
components=[TestPopulation(), model, mortality],
configuration=base_config,
plugin_configuration=base_plugins)
# Move everyone into the event state
simulation.step()
assert np.all(simulation.get_population()[disease] == 'event')
simulation.step()
# Not enough time has passed for people to move out of the event state, so they should all still be there
assert np.all(simulation.get_population()[disease] == 'event')
simulation.step()
# Make sure some people have died and remained in event state
assert (simulation.get_population()['alive'] == 'alive').sum() < pop_size
assert ((simulation.get_population()['alive'] == 'dead').sum() == (
simulation.get_population()[disease] == 'event').sum())
# enough time has passed so living people should transition away to sick
assert ((simulation.get_population()['alive'] == 'alive').sum() == (
simulation.get_population()[disease] == 'sick').sum())
def crude_birth_rate_data(live_births=500):
return (build_table(['mean_value', live_births], 1990, 2017, ('age', 'year', 'sex', 'parameter', 'value'))
.query('age_start == 25 and sex != "Both"')
.drop(['age_start', 'age_end'], 'columns'))
def test_subset_rows_extra_filters():
data = build_table(1, 1990, 2010)
with pytest.raises(ValueError):
_subset_rows(data, missing_thing=12)
def test_interpolated_tables(base_config):
year_start = base_config.time.start.year
year_end = base_config.time.end.year
years = build_table(lambda age, sex, year: year, year_start, year_end)
ages = build_table(lambda age, sex, year: age, year_start, year_end)
one_d_age = ages.copy()
del one_d_age['year']
one_d_age = one_d_age.drop_duplicates()
base_config.update({
'population': {
'population_size': 10000
},
'interpolation': {
'order': 1
}
}) # the results we're checking later assume interp order 1
simulation = InteractiveContext(components=[TestPopulation()],
configuration=base_config)
manager = simulation._tables
years = manager.build_table(years,
key_columns=('sex', ),
parameter_columns=(
'age',
'year',
),
value_columns=None)
ages = manager.build_table(ages,
key_columns=('sex', ),
parameter_columns=(
'age',
'year',
),
value_columns=None)
one_d_age = manager.build_table(one_d_age,
key_columns=('sex', ),
parameter_columns=('age', ),
value_columns=None)
pop = simulation.get_population(untracked=True)
result_years = years(pop.index)
result_ages = ages(pop.index)
result_ages_1d = one_d_age(pop.index)
fractional_year = simulation._clock.time.year
fractional_year += simulation._clock.time.timetuple().tm_yday / 365.25
assert np.allclose(result_years, fractional_year)
assert np.allclose(result_ages, pop.age)
assert np.allclose(result_ages_1d, pop.age)
simulation._clock._time += pd.Timedelta(30.5 * 125, unit='D')
simulation._population._population.age += 125 / 12
result_years = years(pop.index)
result_ages = ages(pop.index)
result_ages_1d = one_d_age(pop.index)
fractional_year = simulation._clock.time.year
fractional_year += simulation._clock.time.timetuple().tm_yday / 365.25
assert np.allclose(result_years, fractional_year)
assert np.allclose(result_ages, pop.age)
assert np.allclose(result_ages_1d, pop.age)