def run_calibration_chain(max_seconds: int, run_id: int, num_chains: int):
targets = load_targets("sir_example", Region.VICTORIA)
params = load_params("sir_example", Region.VICTORIA)
prevalence_infectious = targets["prevalence_infectious"]
target_outputs = [
{
"output_key": prevalence_infectious["output_key"],
"years": prevalence_infectious["times"],
"values": prevalence_infectious["values"],
"loglikelihood_distri": "normal",
"time_weights": list(range(1, len(prevalence_infectious["times"]) + 1)),
},
]
calib = Calibration(
"sir_example",
build_model,
params,
PRIORS,
target_outputs,
run_id,
num_chains,
region_name=Region.AUSTRALIA,
)
calib.run_fitting_algorithm(
run_mode="autumn_mcmc",
n_chains=1,
available_time=max_seconds,
)
def run_calibration_chain(max_seconds: int, run_id: int, num_chains: int):
params = load_params("sir_example", Region.PHILIPPINES)
calib = Calibration(
"sir_example",
build_model,
params,
PRIORS,
TARGET_OUTPUTS,
run_id,
num_chains,
region_name=Region.PHILIPPINES,
)
calib.run_fitting_algorithm(
run_mode="autumn_mcmc",
n_chains=1,
available_time=max_seconds,
)
def run_calibration_chain(max_seconds: int, run_id: int, num_chains: int):
params = load_params("tuberculosis_strains", Region.PHILIPPINES)
calib = Calibration(
"tuberculosis_strains",
build_model,
params,
PRIORS,
TARGET_OUTPUTS,
run_id,
num_chains,
region_name=Region.PHILIPPINES,
)
calib.run_fitting_algorithm(
run_mode="autumn_mcmc",
n_chains=1,
available_time=max_seconds,
haario_scaling_factor=params["default"]["haario_scaling_factor"],
)
def run_calibration_chain(max_seconds: int, run_id: int, num_chains: int):
params = load_params("tuberculosis", Region.MARSHALL_ISLANDS)
calib = Calibration(
"tuberculosis",
build_model,
params,
PRIORS,
TARGET_OUTPUTS,
run_id,
num_chains,
region_name=Region.MARSHALL_ISLANDS,
initialisation_type=params["default"]["metropolis_initialisation"],
metropolis_init_rel_step_size=0.10,
)
calib.run_fitting_algorithm(
run_mode="autumn_mcmc",
n_chains=1,
available_time=max_seconds,
haario_scaling_factor=params["default"]["haario_scaling_factor"],
)
def params(self):
"""Returns the model parameters for the given region"""
return load_params(self.app_name, self.region_name)
def powerbi_postprocess(source_db_path: str, target_db_path: str, run_id: str):
"""
Read the model outputs from a database and then convert them into a form
that is readable by our PowerBI dashboard.
Save the converted data into its own database.
"""
source_db = get_database(source_db_path)
target_db = get_database(target_db_path)
tables_to_copy = [t for t in source_db.table_names() if t != "outputs"]
for table_name in tables_to_copy:
logger.info("Copying %s", table_name)
table_df = source_db.query(table_name)
if table_name == "uncertainty":
# Rename "time" field to "times"
table_df.rename(columns={"time": "times"})
target_db.dump_df(table_name, table_df)
app_name, region_name, timestamp, git_commit = read_run_id(run_id)
# Add build metadata table
build_key = f"{timestamp}-{git_commit}"
logger.info("Adding 'build' metadata table with key %s", build_key)
build_df = pd.DataFrame.from_dict({
"build_key": [build_key],
"app_name": [app_name],
"region_name": [region_name]
})
target_db.dump_df("build", build_df)
# Add scenario metadata table
logger.info("Adding 'scenario' metadata table")
params = load_params(app_name, region_name)
# Add default scenario
scenario_data = [{
"scenario": 0,
"start_time": int(params["default"]["time"]["start"]),
"description": params["default"].get("description", ""),
}]
for sc_idx, sc_params in params["scenarios"].items():
sc_datum = {
"scenario": int(sc_idx),
"start_time": int(sc_params["time"]["start"]),
"description": sc_params.get("description", ""),
}
scenario_data.append(sc_datum)
scenario_df = pd.DataFrame(scenario_data)
target_db.dump_df("scenario", scenario_df)
# Add calibration targets
logger.info("Adding 'targets' table")
targets = load_targets(app_name, region_name)
targets_data = []
for target in targets.values():
for t, v in zip(target["times"], target["values"]):
t_datum = {
"key": target["output_key"],
"times": t,
"value": v,
}
targets_data.append(t_datum)
targets_df = pd.DataFrame(targets_data)
target_db.dump_df("targets", targets_df)
logger.info("Converting outputs to PowerBI format")
outputs_df = source_db.query("outputs")
pbi_outputs_df = unpivot_outputs(outputs_df)
target_db.dump_df("powerbi_outputs", pbi_outputs_df)
logger.info("Finished creating PowerBI output database at %s",
target_db_path)
def multi_country_cdr(plotter, calib_dir_path, mcmc_tables, mcmc_params,
targets, app_name, region_name):
"""
Code taken directly from the fit calibration file at this stage.
"""
from dash.dashboards.calibration_results.plots import get_cdr_constants
param_name = "testing_to_detection.assumed_cdr_parameter"
start_date = st.sidebar.slider("Start date", 1, 365, 1)
end_date = st.sidebar.slider("End date", 1, 365, 275)
samples = st.sidebar.slider("Samples", 1, 200, 10)
label_rotation = st.sidebar.slider("Label rotation", 0, 90, 0)
detected_proportions = []
# Get data for plotting
for i_region in range(len(region_name)):
# Extract parameters relevant to this function
region = region_name[i_region].replace("-", "_")
params = load_params(app_name, region)
(
iso3,
testing_year,
assumed_tests_parameter,
smoothing_period,
agegroup_params,
time_params,
times,
agegroup_strata,
) = get_cdr_constants(params["default"])
pop_region = params["default"]["population"]["region"]
pop_year = params["default"]["population"]["year"]
# Collate parameters into one structure
testing_to_detection_values = []
for i_chain in range(len(mcmc_params)):
param_mask = mcmc_params[i_chain][0]["name"] == param_name
testing_to_detection_values += mcmc_params[i_chain][0]["value"][
param_mask].tolist()
sampled_test_to_detect_vals = random.sample(
testing_to_detection_values, samples)
# Get CDR function - needs to be done outside of autumn, because it is importing from the apps
testing_region = "Victoria" if iso3 == "AUS" else pop_region
testing_year = 2020 if iso3 == "AUS" else pop_year
testing_pops = inputs.get_population_by_agegroup(agegroup_strata,
iso3,
testing_region,
year=testing_year)
detected_proportions.append([])
for assumed_cdr_parameter in sampled_test_to_detect_vals:
detected_proportions[i_region].append(
find_cdr_function_from_test_data(
assumed_tests_parameter,
assumed_cdr_parameter,
smoothing_period,
iso3,
testing_pops,
subregion=testing_region,
))
plots.calibration.plots.plot_multi_cdr_curves(plotter, times,
detected_proportions,
start_date, end_date,
label_rotation, region_name)
def plot_cdr_curves(
plotter: StreamlitPlotter,
calib_dir_path: str,
mcmc_tables: List[pd.DataFrame],
mcmc_params: List[pd.DataFrame],
targets: dict,
app_name: str,
region: str,
):
start_date, end_date = STANDARD_X_LIMITS
samples, label_rotation, region_name = 70, 90, "victoria"
params = load_params(app_name, region_name)
(
iso3,
testing_year,
assumed_tests_parameter,
smoothing_period,
agegroup_params,
time_params,
times,
agegroup_strata,
) = get_cdr_constants(params["default"])
# Collate parameters into one structure
testing_to_detection_values = []
for i_chain in range(len(mcmc_params)):
param_mask = mcmc_params[i_chain][
"name"] == "testing_to_detection.assumed_cdr_parameter"
testing_to_detection_values += mcmc_params[i_chain]["value"][
param_mask].tolist()
# Sample testing values from all the ones available, to avoid plotting too many curves
if samples > len(testing_to_detection_values):
st.write(
"Warning: Requested samples greater than detection values estimated"
)
samples = len(testing_to_detection_values)
sampled_test_to_detect_vals = random.sample(testing_to_detection_values,
samples)
pop = Population
country = Country
country.iso3 = "AUS"
testing_pop, testing_region = get_testing_pop(agegroup_strata, country,
pop)
detected_proportion = []
for assumed_cdr_parameter in sampled_test_to_detect_vals:
detected_proportion.append(
find_cdr_function_from_test_data(
assumed_tests_parameter,
assumed_cdr_parameter,
smoothing_period,
iso3,
testing_pop,
))
plots.calibration.plots.plot_cdr_curves(
plotter,
times,
detected_proportion,
end_date,
label_rotation,
start_date=start_date,
alpha=0.1,
line_width=1.5,
)
def plot_cdr_curves(
plotter: StreamlitPlotter,
calib_dir_path: str,
mcmc_tables: List[pd.DataFrame],
mcmc_params: List[pd.DataFrame],
targets: dict,
app_name: str,
region: str,
):
param_name = "testing_to_detection.assumed_cdr_parameter"
region_name = region.replace("-", "_")
end_date = st.sidebar.slider("End date", 1, 365, 275)
samples = st.sidebar.slider("Samples", 1, 200, 10)
label_rotation = st.sidebar.slider("Label rotation", 0, 90, 0)
# Extract parameters relevant to this function
params = load_params(app_name, region_name)
(
iso3,
testing_year,
assumed_tests_parameter,
smoothing_period,
agegroup_params,
time_params,
times,
agegroup_strata,
) = get_cdr_constants(params["default"])
# Collate parameters into one structure
testing_to_detection_values = []
for i_chain in range(len(mcmc_params)):
param_mask = mcmc_params[i_chain]["name"] == param_name
testing_to_detection_values += mcmc_params[i_chain]["value"][
param_mask].tolist()
# Sample testing values from all the ones available, to avoid plotting too many curves
if samples > len(testing_to_detection_values):
st.write(
"Warning: Requested samples greater than detection values estimated"
)
samples = len(testing_to_detection_values)
sampled_test_to_detect_vals = random.sample(testing_to_detection_values,
samples)
# Get CDR function - needs to be done outside of autumn, because it is importing from the apps
testing_pops = inputs.get_population_by_agegroup(agegroup_strata,
iso3,
None,
year=testing_year)
if iso3 == "AUS":
st.write("WARNING - testing populations are not correct for Victoria")
detected_proportion = []
for assumed_cdr_parameter in sampled_test_to_detect_vals:
detected_proportion.append(
find_cdr_function_from_test_data(
assumed_tests_parameter,
assumed_cdr_parameter,
smoothing_period,
iso3,
testing_pops,
))
plots.calibration.plots.plot_cdr_curves(plotter, times,
detected_proportion, end_date,
label_rotation)