def load_observations(fips=None, ref_date=REF_DATE):
"""
Load observations (new cases, new deaths and hospitalizations) for
given fips code.
Parameters
----------
fips: str
FIPS code.
ref_date: Datetime
Reference start date.
Returns
-------
observations: pd.DataFrame
Contains observations for given fips codes, with columns:
- new_cases: float, observed new cases
- new_deaths: float, observed new deaths
- hospitalizations: float, observed hospitalizatons
and dates of observation as index.
"""
observations = {}
if len(fips) == 5:
times, observations['new_cases'], observations['new_deaths'] = \
load_data.load_new_case_data_by_fips(fips, ref_date)
hospital_times, hospitalizations, hospitalization_data_type = \
load_data.load_hospitalization_data(fips, t0=ref_date)
observations['times'] = times.values
elif len(fips) == 2:
state_obj = us.states.lookup(fips)
observations['times'], observations['new_cases'], observations['new_deaths'] = \
load_data.load_new_case_data_by_state(state_obj.name, ref_date)
hospital_times, hospitalizations, hospitalization_data_type = \
load_data.load_hospitalization_data_by_state(state_obj.abbr, t0=ref_date)
observations['times'] = np.array(observations['times'])
observations['hospitalizations'] = np.full(
observations['times'].shape[0], np.nan)
if hospitalization_data_type is HospitalizationDataType.CUMULATIVE_HOSPITALIZATIONS:
observations['hospitalizations'][
hospital_times -
observations['times'].min()] = np.diff(hospitalizations)
elif hospitalization_data_type is HospitalizationDataType.CURRENT_HOSPITALIZATIONS:
observations['hospitalizations'][
hospital_times -
observations['times'].min()] = hospitalizations
observation_dates = [
ref_date + timedelta(int(t)) for t in observations['times']
]
observations = pd.DataFrame(
observations,
index=pd.DatetimeIndex(observation_dates)).dropna(axis=1,
how='all')
return observations
def __init__(self,
fips,
ref_date=datetime(year=2020, month=1, day=1),
min_deaths=2,
n_years=1,
cases_to_deaths_err_factor=.5,
hospital_to_deaths_err_factor=.5,
percent_error_on_max_observation=0.5,
with_age_structure=False):
# Seed the random state. It is unclear whether this propagates to the
# Minuit optimizer.
np.random.seed(seed=42)
self.fips = fips
self.ref_date = ref_date
self.min_deaths = min_deaths
self.t_list = np.linspace(0, int(365 * n_years),
int(365 * n_years) + 1)
self.cases_to_deaths_err_factor = cases_to_deaths_err_factor
self.hospital_to_deaths_err_factor = hospital_to_deaths_err_factor
self.percent_error_on_max_observation = percent_error_on_max_observation
self.t0_guess = 60
self.with_age_structure = with_age_structure
if len(fips) == 2: # State FIPS are 2 digits
self.agg_level = AggregationLevel.STATE
self.state_obj = us.states.lookup(self.fips)
self.state = self.state_obj.name
self.times, self.observed_new_cases, self.observed_new_deaths = \
load_data.load_new_case_data_by_state(self.state, self.ref_date)
self.hospital_times, self.hospitalizations, self.hospitalization_data_type = \
load_data.load_hospitalization_data_by_state(self.state_obj.abbr, t0=self.ref_date)
self.display_name = self.state
else:
self.agg_level = AggregationLevel.COUNTY
geo_metadata = load_data.load_county_metadata().set_index(
'fips').loc[fips].to_dict()
state = geo_metadata['state']
self.state_obj = us.states.lookup(state)
county = geo_metadata['county']
if county:
self.display_name = county + ', ' + state
else:
self.display_name = state
# TODO Swap for new data source.
self.times, self.observed_new_cases, self.observed_new_deaths = \
load_data.load_new_case_data_by_fips(self.fips, t0=self.ref_date)
self.hospital_times, self.hospitalizations, self.hospitalization_data_type = \
load_data.load_hospitalization_data(self.fips, t0=self.ref_date)
self.cases_stdev, self.hosp_stdev, self.deaths_stdev = self.calculate_observation_errors(
)
self.set_inference_parameters()
self.model_fit_keys = ['R0', 'eps', 't_break', 'log10_I_initial']
self.SEIR_kwargs = self.get_average_seir_parameters()
self.fit_results = None
self.mle_model = None
self.chi2_deaths = None
self.chi2_cases = None
self.chi2_hosp = None
self.dof_deaths = None
self.dof_cases = None
self.dof_hosp = None
def __init__(
self,
fips,
window_size=InferRtConstants.COUNT_SMOOTHING_WINDOW_SIZE,
kernel_std=5,
r_list=np.linspace(0, 10, 501),
process_sigma=0.05,
ref_date=datetime(year=2020, month=1, day=1),
confidence_intervals=(0.68, 0.95),
min_cases=5,
min_deaths=5,
include_testing_correction=True,
):
np.random.seed(InferRtConstants.RNG_SEED)
# Param Generation used for Xcor in align_time_series, has some stochastic FFT elements.
self.fips = fips
self.r_list = r_list
self.window_size = window_size
self.kernel_std = kernel_std
self.process_sigma = process_sigma
self.ref_date = ref_date
self.confidence_intervals = confidence_intervals
self.min_cases = min_cases
self.min_deaths = min_deaths
self.include_testing_correction = include_testing_correction
# Because rounding is disabled we don't need high min_deaths, min_cases anymore
self.min_cases = min(InferRtConstants.MIN_COUNTS_TO_INFER, self.min_cases)
if not InferRtConstants.DISABLE_DEATHS:
self.min_deaths = min(InferRtConstants.MIN_COUNTS_TO_INFER, self.min_deaths)
if len(fips) == 2: # State FIPS are 2 digits
self.agg_level = AggregationLevel.STATE
self.state_obj = us.states.lookup(self.fips)
self.state = self.state_obj.name
(
self.times,
self.observed_new_cases,
self.observed_new_deaths,
) = load_data.load_new_case_data_by_state(
self.state,
self.ref_date,
include_testing_correction=self.include_testing_correction,
)
self.times_raw_new_cases, self.raw_new_cases, _ = load_data.load_new_case_data_by_state(
self.state, self.ref_date, include_testing_correction=False
)
(
self.hospital_times,
self.hospitalizations,
self.hospitalization_data_type,
) = load_data.load_hospitalization_data_by_state(
state=self.state_obj.abbr, t0=self.ref_date
)
self.display_name = self.state
else:
self.agg_level = AggregationLevel.COUNTY
self.geo_metadata = (
load_data.load_county_metadata().set_index("fips").loc[fips].to_dict()
)
self.state = self.geo_metadata["state"]
self.state_obj = us.states.lookup(self.state)
self.county = self.geo_metadata["county"]
if self.county:
self.display_name = self.county + ", " + self.state
else:
self.display_name = self.state
(
self.times,
self.observed_new_cases,
self.observed_new_deaths,
) = load_data.load_new_case_data_by_fips(
self.fips,
t0=self.ref_date,
include_testing_correction=self.include_testing_correction,
)
(
self.times_raw_new_cases,
self.raw_new_cases,
_,
) = load_data.load_new_case_data_by_fips(
self.fips, t0=self.ref_date, include_testing_correction=False,
)
(
self.hospital_times,
self.hospitalizations,
self.hospitalization_data_type,
) = load_data.load_hospitalization_data(self.fips, t0=self.ref_date)
self.case_dates = [ref_date + timedelta(days=int(t)) for t in self.times]
self.raw_new_case_dates = [
ref_date + timedelta(days=int(t)) for t in self.times_raw_new_cases
]
if self.hospitalization_data_type:
self.hospital_dates = [ref_date + timedelta(days=int(t)) for t in self.hospital_times]
self.default_parameters = ParameterEnsembleGenerator(
fips=self.fips, N_samples=500, t_list=np.linspace(0, 365, 366)
).get_average_seir_parameters()
# Serial period = Incubation + 0.5 * Infections
self.serial_period = (
1 / self.default_parameters["sigma"] + 0.5 * 1 / self.default_parameters["delta"]
)
# If we only receive current hospitalizations, we need to account for
# the outflow to reconstruct new admissions.
if (
self.hospitalization_data_type
is load_data.HospitalizationDataType.CURRENT_HOSPITALIZATIONS
):
los_general = self.default_parameters["hospitalization_length_of_stay_general"]
los_icu = self.default_parameters["hospitalization_length_of_stay_icu"]
hosp_rate_general = self.default_parameters["hospitalization_rate_general"]
hosp_rate_icu = self.default_parameters["hospitalization_rate_icu"]
icu_rate = hosp_rate_icu / hosp_rate_general
flow_out_of_hosp = self.hospitalizations[:-1] * (
(1 - icu_rate) / los_general + icu_rate / los_icu
)
# We are attempting to reconstruct the cumulative hospitalizations.
self.hospitalizations = np.diff(self.hospitalizations) + flow_out_of_hosp
self.hospital_dates = self.hospital_dates[1:]
self.hospital_times = self.hospital_times[1:]
self.log_likelihood = None
self.log = structlog.getLogger(Rt_Inference_Target=self.display_name)
self.log.info(event="Running:")
def map_fips(self, fips):
"""
For a given county fips code, generate the CAN UI output format.
Parameters
----------
fips: str
County FIPS code to map.
"""
logging.info(f"Mapping output to WebUI for {self.state}, {fips}")
pyseir_outputs = load_data.load_ensemble_results(fips)
if len(fips) == 5 and fips not in self.df_whitelist.fips.values:
logging.info(f"Excluding {fips} due to white list...")
return
try:
fit_results = load_inference_result(fips)
t0_simulation = datetime.fromisoformat(fit_results["t0_date"])
except (KeyError, ValueError):
logging.error(f"Fit result not found for {fips}. Skipping...")
return
# ---------------------------------------------------------------------
# Rescale hosps based on the population ratio... Could swap this to
# infection ratio later?
# ---------------------------------------------------------------------
hosp_times, current_hosp, _ = load_data.load_hospitalization_data_by_state(
state=self.state_abbreviation,
t0=t0_simulation,
convert_cumulative_to_current=True,
category="hospitalized",
)
_, current_icu, _ = load_data.load_hospitalization_data_by_state(
state=self.state_abbreviation,
t0=t0_simulation,
convert_cumulative_to_current=True,
category="icu",
)
if len(fips) == 5:
population = self.population_data.get_record_for_fips(fips)[CommonFields.POPULATION]
else:
population = self.population_data.get_record_for_state(self.state_abbreviation)[
CommonFields.POPULATION
]
# logging.info(f'Mapping output to WebUI for {self.state}, {fips}')
# pyseir_outputs = load_data.load_ensemble_results(fips)
# if pyseir_outputs is None:
# logging.warning(f'Fit result not found for {fips}: Skipping county')
# return None
policies = [key for key in pyseir_outputs.keys() if key.startswith("suppression_policy")]
if current_hosp is not None:
t_latest_hosp_data, current_hosp = hosp_times[-1], current_hosp[-1]
t_latest_hosp_data_date = t0_simulation + timedelta(days=int(t_latest_hosp_data))
state_hosp_gen = load_data.get_compartment_value_on_date(
fips=fips[:2], compartment="HGen", date=t_latest_hosp_data_date
)
state_hosp_icu = load_data.get_compartment_value_on_date(
fips=fips[:2], compartment="HICU", date=t_latest_hosp_data_date
)
if len(fips) == 5:
# Rescale the county level hospitalizations by the expected
# ratio of county / state hospitalizations from simulations.
# We use ICU data if available too.
county_hosp = load_data.get_compartment_value_on_date(
fips=fips,
compartment="HGen",
date=t_latest_hosp_data_date,
ensemble_results=pyseir_outputs,
)
county_icu = load_data.get_compartment_value_on_date(
fips=fips,
compartment="HICU",
date=t_latest_hosp_data_date,
ensemble_results=pyseir_outputs,
)
current_hosp *= (county_hosp + county_icu) / (state_hosp_gen + state_hosp_icu)
hosp_rescaling_factor = current_hosp / (state_hosp_gen + state_hosp_icu)
# Some states have covidtracking issues. We shouldn't ground ICU cases
# to zero since so far these have all been bad reporting.
if current_icu is not None and current_icu[-1] > 0:
icu_rescaling_factor = current_icu[-1] / state_hosp_icu
else:
icu_rescaling_factor = current_hosp / (state_hosp_gen + state_hosp_icu)
else:
hosp_rescaling_factor = 1.0
icu_rescaling_factor = 1.0
# Iterate through each suppression policy.
# Model output is interpolated to the dates desired for the API.
for i_policy, suppression_policy in enumerate(
[key for key in pyseir_outputs.keys() if key.startswith("suppression_policy")]
):
output_for_policy = pyseir_outputs[suppression_policy]
output_model = pd.DataFrame()
t_list = output_for_policy["t_list"]
t_list_downsampled = range(0, int(max(t_list)), self.output_interval_days)
output_model[schema.DAY_NUM] = t_list_downsampled
output_model[schema.DATE] = [
(t0_simulation + timedelta(days=t)).date().strftime("%m/%d/%y")
for t in t_list_downsampled
]
output_model[schema.TOTAL] = population
output_model[schema.TOTAL_SUSCEPTIBLE] = np.interp(
t_list_downsampled, t_list, output_for_policy["S"]["ci_50"]
)
output_model[schema.EXPOSED] = np.interp(
t_list_downsampled, t_list, output_for_policy["E"]["ci_50"]
)
output_model[schema.INFECTED] = np.interp(
t_list_downsampled,
t_list,
np.add(output_for_policy["I"]["ci_50"], output_for_policy["A"]["ci_50"]),
) # Infected + Asympt.
output_model[schema.INFECTED_A] = output_model[schema.INFECTED]
output_model[schema.INFECTED_B] = hosp_rescaling_factor * np.interp(
t_list_downsampled, t_list, output_for_policy["HGen"]["ci_50"]
) # Hosp General
output_model[schema.INFECTED_C] = icu_rescaling_factor * np.interp(
t_list_downsampled, t_list, output_for_policy["HICU"]["ci_50"]
) # Hosp ICU
# General + ICU beds. don't include vent here because they are also counted in ICU
output_model[schema.ALL_HOSPITALIZED] = np.add(
output_model[schema.INFECTED_B], output_model[schema.INFECTED_C]
)
output_model[schema.ALL_INFECTED] = output_model[schema.INFECTED]
output_model[schema.DEAD] = np.interp(
t_list_downsampled, t_list, output_for_policy["total_deaths"]["ci_50"]
)
final_beds = np.mean(output_for_policy["HGen"]["capacity"])
output_model[schema.BEDS] = final_beds
output_model[schema.CUMULATIVE_INFECTED] = np.interp(
t_list_downsampled,
t_list,
np.cumsum(output_for_policy["total_new_infections"]["ci_50"]),
)
if fit_results:
output_model[schema.Rt] = np.interp(
t_list_downsampled,
t_list,
fit_results["eps"] * fit_results["R0"] * np.ones(len(t_list)),
)
output_model[schema.Rt_ci90] = np.interp(
t_list_downsampled,
t_list,
2 * fit_results["eps_error"] * fit_results["R0"] * np.ones(len(t_list)),
)
else:
output_model[schema.Rt] = 0
output_model[schema.Rt_ci90] = 0
output_model[schema.CURRENT_VENTILATED] = icu_rescaling_factor * np.interp(
t_list_downsampled, t_list, output_for_policy["HVent"]["ci_50"]
)
output_model[schema.POPULATION] = population
# Average capacity.
output_model[schema.ICU_BED_CAPACITY] = np.mean(output_for_policy["HICU"]["capacity"])
output_model[schema.VENTILATOR_CAPACITY] = np.mean(
output_for_policy["HVent"]["capacity"]
)
# Truncate date range of output.
output_dates = pd.to_datetime(output_model["date"])
output_model = output_model[
(output_dates >= datetime(month=3, day=3, year=2020))
& (output_dates < datetime.today() + timedelta(days=90))
]
output_model = output_model.fillna(0)
# Fill in results for the Rt indicator.
try:
rt_results = load_Rt_result(fips)
rt_results.index = rt_results["Rt_MAP_composite"].index.strftime("%m/%d/%y")
merged = output_model.merge(
rt_results[["Rt_MAP_composite", "Rt_ci95_composite"]],
right_index=True,
left_on="date",
how="left",
)
output_model[schema.RT_INDICATOR] = merged["Rt_MAP_composite"]
# With 90% probability the value is between rt_indicator - ci90 to rt_indicator + ci90
output_model[schema.RT_INDICATOR_CI90] = (
merged["Rt_ci95_composite"] - merged["Rt_MAP_composite"]
)
except (ValueError, KeyError) as e:
output_model[schema.RT_INDICATOR] = "NaN"
output_model[schema.RT_INDICATOR_CI90] = "NaN"
output_model[[schema.RT_INDICATOR, schema.RT_INDICATOR_CI90]] = output_model[
[schema.RT_INDICATOR, schema.RT_INDICATOR_CI90]
].fillna("NaN")
# Truncate floats and cast as strings to match data model.
int_columns = [
col
for col in output_model.columns
if col
not in (
schema.DATE,
schema.Rt,
schema.Rt_ci90,
schema.RT_INDICATOR,
schema.RT_INDICATOR_CI90,
)
]
output_model.loc[:, int_columns] = (
output_model[int_columns].fillna(0).astype(int).astype(str)
)
output_model.loc[
:, [schema.Rt, schema.Rt_ci90, schema.RT_INDICATOR, schema.RT_INDICATOR_CI90]
] = (
output_model[
[schema.Rt, schema.Rt_ci90, schema.RT_INDICATOR, schema.RT_INDICATOR_CI90]
]
.fillna(0)
.round(decimals=4)
.astype(str)
)
# Convert the records format to just list(list(values))
output_model = [
[val for val in timestep.values()]
for timestep in output_model.to_dict(orient="records")
]
output_path = get_run_artifact_path(
fips, RunArtifact.WEB_UI_RESULT, output_dir=self.output_dir
)
policy_enum = Intervention.from_webui_data_adaptor(suppression_policy)
output_path = output_path.replace("__INTERVENTION_IDX__", str(policy_enum.value))
with open(output_path, "w") as f:
json.dump(output_model, f)
def __init__(self,
fips,
window_size=7,
kernel_std=2,
r_list=np.linspace(0, 10, 501),
process_sigma=0.15,
ref_date=datetime(year=2020, month=1, day=1),
confidence_intervals=(0.68, 0.75, 0.90)):
self.fips = fips
self.r_list = r_list
self.window_size = window_size
self.kernel_std = kernel_std
self.process_sigma = process_sigma
self.ref_date = ref_date
self.confidence_intervals = confidence_intervals
if len(fips) == 2: # State FIPS are 2 digits
self.agg_level = AggregationLevel.STATE
self.state_obj = us.states.lookup(self.fips)
self.state = self.state_obj.name
self.geo_metadata = load_data.load_county_metadata_by_state(self.state).loc[self.state].to_dict()
self.times, self.observed_new_cases, self.observed_new_deaths = \
load_data.load_new_case_data_by_state(self.state, self.ref_date)
self.hospital_times, self.hospitalizations, self.hospitalization_data_type = \
load_data.load_hospitalization_data_by_state(self.state_obj.abbr, t0=self.ref_date)
self.display_name = self.state
else:
self.agg_level = AggregationLevel.COUNTY
self.geo_metadata = load_data.load_county_metadata().set_index('fips').loc[fips].to_dict()
self.state = self.geo_metadata['state']
self.state_obj = us.states.lookup(self.state)
self.county = self.geo_metadata['county']
if self.county:
self.display_name = self.county + ', ' + self.state
else:
self.display_name = self.state
# TODO Swap for new data source.
self.times, self.observed_new_cases, self.observed_new_deaths = \
load_data.load_new_case_data_by_fips(self.fips, t0=self.ref_date)
self.hospital_times, self.hospitalizations, self.hospitalization_data_type = \
load_data.load_hospitalization_data(self.fips, t0=self.ref_date)
logging.info(f'Running Rt Inference for {self.display_name}')
self.case_dates = [ref_date + timedelta(days=int(t)) for t in self.times]
if self.hospitalization_data_type:
self.hospital_dates = [ref_date + timedelta(days=int(t)) for t in self.hospital_times]
self.default_parameters = ParameterEnsembleGenerator(
fips=self.fips,
N_samples=500,
t_list=np.linspace(0, 365, 366)
).get_average_seir_parameters()
# Serial period = Incubation + 0.5 * Infections
self.serial_period = 1 / self.default_parameters['sigma'] + 0.5 * 1 / self.default_parameters['delta']
# If we only receive current hospitalizations, we need to account for
# the outflow to reconstruct new admissions.
if self.hospitalization_data_type is load_data.HospitalizationDataType.CURRENT_HOSPITALIZATIONS:
los_general = self.default_parameters['hospitalization_length_of_stay_general']
los_icu = self.default_parameters['hospitalization_length_of_stay_icu']
hosp_rate_general = self.default_parameters['hospitalization_rate_general']
hosp_rate_icu = self.default_parameters['hospitalization_rate_icu']
icu_rate = hosp_rate_icu / hosp_rate_general
flow_out_of_hosp = self.hospitalizations[:-1] * ((1 - icu_rate) / los_general + icu_rate / los_icu)
# We are attempting to reconstruct the cumulative hospitalizations.
self.hospitalizations = np.diff(self.hospitalizations) + flow_out_of_hosp
self.hospital_dates = self.hospital_dates[1:]
self.hospital_times = self.hospital_times[1:]
self.log_likelihood = None
def __init__(
self,
fips,
ref_date=datetime(year=2020, month=1, day=1),
min_deaths=2,
n_years=1,
cases_to_deaths_err_factor=0.5,
hospital_to_deaths_err_factor=0.5,
percent_error_on_max_observation=0.5,
with_age_structure=False,
):
# Seed the random state. It is unclear whether this propagates to the
# Minuit optimizer.
np.random.seed(seed=42)
self.fips = fips
self.ref_date = ref_date
self.days_since_ref_date = (dt.date.today() - ref_date.date() -
timedelta(days=7)).days
# ndays end of 2nd ramp may extend past days_since_ref_date w/o penalty on chi2 score
self.days_allowed_beyond_ref = 0
self.min_deaths = min_deaths
self.t_list = np.linspace(0, int(365 * n_years),
int(365 * n_years) + 1)
self.cases_to_deaths_err_factor = cases_to_deaths_err_factor
self.hospital_to_deaths_err_factor = hospital_to_deaths_err_factor
self.percent_error_on_max_observation = percent_error_on_max_observation
self.t0_guess = 60
self.with_age_structure = with_age_structure
if len(fips) == 2: # State FIPS are 2 digits
self.agg_level = AggregationLevel.STATE
self.state_obj = us.states.lookup(self.fips)
self.state = self.state_obj.name
(
self.times,
self.observed_new_cases,
self.observed_new_deaths,
) = load_data.load_new_case_data_by_state(self.state,
self.ref_date)
(
self.hospital_times,
self.hospitalizations,
self.hospitalization_data_type,
) = load_data.load_hospitalization_data_by_state(
self.state_obj.abbr, t0=self.ref_date)
(
self.icu_times,
self.icu,
self.icu_data_type,
) = load_data.load_hospitalization_data_by_state(
self.state_obj.abbr,
t0=self.ref_date,
category=HospitalizationCategory.ICU)
self.display_name = self.state
else:
self.agg_level = AggregationLevel.COUNTY
geo_metadata = load_data.load_county_metadata().set_index(
"fips").loc[fips].to_dict()
state = geo_metadata["state"]
self.state_obj = us.states.lookup(state)
county = geo_metadata["county"]
if county:
self.display_name = county + ", " + state
else:
self.display_name = state
# TODO Swap for new data source.
(
self.times,
self.observed_new_cases,
self.observed_new_deaths,
) = load_data.load_new_case_data_by_fips(self.fips,
t0=self.ref_date)
(
self.hospital_times,
self.hospitalizations,
self.hospitalization_data_type,
) = load_data.load_hospitalization_data(self.fips,
t0=self.ref_date)
(
self.icu_times,
self.icu,
self.icu_data_type,
) = load_data.load_hospitalization_data(
self.fips,
t0=self.ref_date,
category=HospitalizationCategory.ICU)
self.cases_stdev, self.hosp_stdev, self.deaths_stdev = self.calculate_observation_errors(
)
self.set_inference_parameters()
self.model_fit_keys = [
"R0",
"eps",
"t_break",
"eps2",
"t_delta_phases",
"log10_I_initial",
]
self.SEIR_kwargs = self.get_average_seir_parameters()
self.fit_results = None
self.mle_model = None
self.chi2_deaths = None
self.chi2_cases = None
self.chi2_hosp = None
self.dof_deaths = None
self.dof_cases = None
self.dof_hosp = None