def create_report_file(param_obj, campaign_obj, demographics_obj,
report_data_obj, report_name, debug):
with open(report_name, "w") as outfile:
success = True
total_timesteps = param_obj[KEY_TOTAL_TIMESTEPS]
start_timestep = param_obj[KEY_START_TIME]
initial_population = demographics_obj[KEY_INITIAL_POPULATION]
rates = campaign_obj[KEY_CAMPAIGN_DIP]
durations = campaign_obj[KEY_CAMPAIGN_DURATIONS]
if not report_data_obj: # todo: maybe use try
success = False
outfile.write("BAD: There is no data in the InsetChart report")
else:
new_infections = report_data_obj[KEY_NEW_INFECTIONS]
statistical_population = report_data_obj[
KEY_STATISTICAL_POPULATION]
length = len(rates)
start_duration = start_timestep
new_infections_dict = {}
calculate_new_population = initial_population
for i in range(length):
rate = rates[i]
duration = durations[i]
calculate_new_population = rate * duration + calculate_new_population
end_duration = duration + start_duration
if rate not in new_infections_dict:
new_infections_dict[rate] = []
for j in range(start_duration + 1, end_duration + 1):
if j < total_timesteps + start_timestep:
new_infections_dict[rate].append(new_infections[j])
j += 1
else:
break
if end_duration > total_timesteps + start_timestep:
calculate_new_population -= rate * (
end_duration - total_timesteps - start_timestep)
break
start_duration = end_duration
if end_duration < total_timesteps + start_timestep:
rate = 0.0
if rate not in new_infections_dict:
new_infections_dict[rate] = []
for j in range(end_duration + 1, len(new_infections)):
new_infections_dict[rate].append(new_infections[j])
with open("new_infections_parsed.json", "w") as file:
json.dump(new_infections_dict, file, indent=4)
# test statistical population channel
diff_population = math.fabs(calculate_new_population -
statistical_population[-1])
if debug:
print( "calculated population is {0}, statistical population " \
"from InsetChart is {1}.".format(calculate_new_population,
statistical_population[-1]) )
error_tolerance = math.fabs(calculate_new_population -
initial_population) * 0.1
if debug:
print("diff_population is {0}, error_tolerance is {1}".format(
diff_population, error_tolerance))
if diff_population > error_tolerance:
success = False
outfile.write(
"BAD: statistical population is {0}, expected about {1}.\n"
.format(statistical_population[-1],
calculate_new_population))
# test poisson distribution for new infections
for rate in new_infections_dict:
dist = new_infections_dict[rate]
title = "rate = " + str(rate)
result = sft.test_poisson(dist,
rate,
route=title,
report_file=outfile,
normal_approximation=False)
# print result, rate, len(dist)
if not result:
success = False
outfile.write(
"BAD: ks poisson test for {0} is {1}.\n".format(
title, result))
numpy_distro = np.random.poisson(rate, len(dist))
sft.plot_data(
dist,
numpy_distro,
title="new infections for {}".format(title),
label1="new infection from model, {}".format(title),
label2="Poisson distro from numpy",
xlabel="data points",
ylabel="new infection",
category="plot_data_{0}".format(title),
show=True)
sft.plot_probability(
dist,
numpy_distro,
title="probability mass function for {}".format(title),
label1="new infection probability from model",
label2="new infection probability from numpy distro",
category="plot_probability_{0}".format(title),
show=True)
outfile.write(sft.format_success_msg(success))
if debug:
print("SUMMARY: Success={0}\n".format(success))
return success
def create_report_file(param_obj, multipliers, infectiousness, report_name, debug):
with open(report_name, "w") as outfile:
success = True
if not multipliers:
outfile.write(sft.sft_no_test_data)
sigma = param_obj[Param_keys.LOGNORMAL_SCALE]
base_infectivity = param_obj[Param_keys.BASE_INFECTIVITY]
if sigma > 0:
mu = - sigma**2 / 2.0
# test log_normal distribution
success = sft.test_lognorm(multipliers,mu=mu, sigma=sigma,report_file=outfile,round = False)
# test mean_l = 1
mean_l = np.mean(multipliers)
mean_infectiousness = np.mean(infectiousness)
outfile.write("mean of the multipliers is {}, expected 1.0.\n".format(mean_l))
outfile.write("mean of the Infectiousness is {0}, while base infectivity is {1}.\n".format(mean_infectiousness,
base_infectivity))
tolerance = 2e-2
if math.fabs(mean_l - 1.0) > tolerance:
outfile.write("BAD: mean of the multipliers is {}, expected 1.0.\n".format(mean_l))
success = False
# plotting
size = len(multipliers)
outfile.write("size is {}\n".format(size))
scale = math.exp(mu)
dist_lognormal = stats.lognorm.rvs(sigma, 0, scale, size)
sft.plot_data(multipliers, dist_lognormal,
label1="Emod", label2="Scipy",
ylabel="Multiplier", xlabel="data point",
category="Emod_vs_Scipy",
title="Emod_vs_Scipy, sigma = {}".format(sigma),
show=True)
sft.plot_probability(multipliers, dist_lognormal,
precision=1, label1="Emod", label2="Scipy",
category="Probability_mass_function_Emod_vs_Scipy",
title="Emod_vs_Scipy, sigma = {}".format(sigma),
show=True)
sft.plot_cdf(multipliers,dist_lognormal,label1="Emod", label2="Scipy",
category="cdf",
title="cdf, sigma = {}".format(sigma),
show=True, line = False)
if debug:
with open("scipy_data.txt", "w") as file:
for n in sorted(dist_lognormal):
file.write(str(n) + "\n")
with open("emod_data.txt", "w") as file:
for n in sorted(multipliers):
file.write(str(n) + "\n")
else:
# sigma = 0, this feature is disabled
for multiplier in multipliers:
if multiplier != 1.0:
success = False
outfile.write("BAD: multiplier is {0} when {1} set to {2}, expected 1.0.\n".format(multiplier, Param_keys.LOGNORMAL_SCALE, sigma))
# plotting
sft.plot_data(multipliers, label1="Multiplier", label2="NA",
category="Multiplier", title="Multiplier_Sigma={}".format(sigma),
ylabel="Multiplier", xlabel="data point",
show=True)
sft.plot_data(infectiousness, label1="Infectiousness",
label2="NA",category="Infectiousness",
title="Infectiousness_Sigma={0}_BaseInfectivity={1}".format(sigma,base_infectivity),
ylabel="Infectiousness",xlabel="data point",
show=True)
outfile.write(sft.format_success_msg(success))
if debug:
print "SUMMARY: Success={0}\n".format(success)
return success
def create_report_file(param_obj, multipliers, infectiousness, report_name, debug):
with open(report_name, "w") as outfile:
success = True
if not multipliers:
outfile.write(sft.sft_no_test_data)
sigma = param_obj[Param_keys.LOGNORMAL_SCALE]
base_infectivity = param_obj[Param_keys.BASE_INFECTIVITY]
if sigma > 0:
mu = - sigma**2 / 2.0
# test log_normal distribution
success = sft.test_lognorm(multipliers,mu=mu, sigma=sigma,report_file=outfile,round = False)
# test mean_l = 1
mean_l = np.mean(multipliers)
mean_infectiousness = np.mean(infectiousness)
outfile.write("mean of the multipliers is {}, expected 1.0.\n".format(mean_l))
outfile.write("mean of the Infectiousness is {0}, while base infectivity is {1}.\n".format(mean_infectiousness,
base_infectivity))
tolerance = 2e-2
if math.fabs(mean_l - 1.0) > tolerance:
outfile.write("BAD: mean of the multipliers is {}, expected 1.0.\n".format(mean_l))
success = False
# plotting
size = len(multipliers)
outfile.write("size is {}\n".format(size))
scale = math.exp(mu)
dist_lognormal = stats.lognorm.rvs(sigma, 0, scale, size)
sft.plot_data_sorted(multipliers, dist_lognormal,
label1="Emod", label2="Scipy",
ylabel="Multiplier", xlabel="data point",
category="Emod_vs_Scipy",
title="Emod_vs_Scipy, sigma = {}".format(sigma),
show=True)
sft.plot_probability(multipliers, dist_lognormal,
precision=1, label1="Emod", label2="Scipy",
category="Probability_mass_function_Emod_vs_Scipy",
title="Emod_vs_Scipy, sigma = {}".format(sigma),
show=True)
sft.plot_cdf(multipliers,dist_lognormal,label1="Emod", label2="Scipy",
category="cdf",
title="cdf, sigma = {}".format(sigma),
show=True, line = False)
if debug:
with open("scipy_data.txt", "w") as file:
for n in sorted(dist_lognormal):
file.write(str(n) + "\n")
with open("emod_data.txt", "w") as file:
for n in sorted(multipliers):
file.write(str(n) + "\n")
else:
# sigma = 0, this feature is disabled
for multiplier in multipliers:
if multiplier != 1.0:
success = False
outfile.write("BAD: multiplier is {0} when {1} set to {2}, expected 1.0.\n".format(multiplier, Param_keys.LOGNORMAL_SCALE, sigma))
# plotting
sft.plot_data_sorted(multipliers, label1="Multiplier", label2="NA",
category="Multiplier", title="Multiplier_Sigma={}".format(sigma),
ylabel="Multiplier", xlabel="data point",
show=True)
sft.plot_data_sorted(infectiousness, label1="Infectiousness",
label2="NA",category="Infectiousness",
title="Infectiousness_Sigma={0}_BaseInfectivity={1}".format(sigma,base_infectivity),
ylabel="Infectiousness",xlabel="data point",
show=True)
outfile.write(sft.format_success_msg(success))
if debug:
print( "SUMMARY: Success={0}\n".format(success) )
return success
def create_report_file(param_obj, campaign_obj, demographics_obj, report_data_obj, report_name, debug):
with open(report_name, "w") as outfile:
success = True
total_timesteps = param_obj[KEY_TOTAL_TIMESTEPS]
start_timestep = param_obj[KEY_START_TIME]
initial_population = demographics_obj[KEY_INITIAL_POPULATION]
rates = campaign_obj[KEY_CAMPAIGN_DIP]
durations = campaign_obj[KEY_CAMPAIGN_DURATIONS]
if not report_data_obj: # todo: maybe use try
success = False
outfile.write("BAD: There is no data in the InsetChart report")
else:
new_infections = report_data_obj[KEY_NEW_INFECTIONS]
statistical_population = report_data_obj[KEY_STATISTICAL_POPULATION]
length = len(rates)
start_duration = start_timestep
new_infections_dict = {}
calculate_new_population = initial_population
for i in range(length):
rate = rates[i]
duration = durations[i]
calculate_new_population = rate * duration + calculate_new_population
end_duration = duration + start_duration
if rate not in new_infections_dict:
new_infections_dict[rate] = []
for j in range(start_duration + 1, end_duration + 1):
if j < total_timesteps + start_timestep:
new_infections_dict[rate].append(new_infections[j])
j += 1
else:
break
if end_duration > total_timesteps + start_timestep:
calculate_new_population -= rate * (end_duration - total_timesteps - start_timestep)
break
start_duration = end_duration
if end_duration < total_timesteps + start_timestep:
rate = 0.0
if rate not in new_infections_dict:
new_infections_dict[rate] = []
for j in range(end_duration + 1, len(new_infections)):
new_infections_dict[rate].append(new_infections[j])
with open("new_infections_parsed.json","w") as file:
json.dump(new_infections_dict, file, indent = 4)
# test statistical population channel
diff_population = math.fabs(calculate_new_population - statistical_population[-1])
if debug:
print "calculated population is {0}, statistical population " \
"from InsetChart is {1}.".format(calculate_new_population,
statistical_population[-1])
error_tolerance = math.fabs(calculate_new_population - initial_population)* 0.1
if debug:
print "diff_population is {0}, error_tolerance is {1}".format(diff_population, error_tolerance)
if diff_population > error_tolerance:
success = False
outfile.write("BAD: statistical population is {0}, expected about {1}.\n".format(statistical_population[-1], calculate_new_population))
# test poisson distribution for new infections
for rate in new_infections_dict:
dist = new_infections_dict[rate]
title = "rate = " + str(rate)
result = sft.test_poisson(dist, rate, route = title, report_file = outfile, normal_approximation = False)
# print result, rate, len(dist)
if not result:
success = False
outfile.write("BAD: ks poisson test for {0} is {1}.\n".format(title, result))
numpy_distro = np.random.poisson(rate, len(dist))
sft.plot_data(dist, sorted(numpy_distro),
title="new infections for {}".format(title),
label1="new infection from model, {}".format(title),
label2="Poisson distro from numpy",
xlabel="data points", ylabel="new infection",
category="plot_data_{0}".format(title), show=True)
sft.plot_probability(dist, numpy_distro,
title="probability mass function for {}".format(title),
label1="new infection probability from model",
label2="new infection probability from numpy distro",
category="plot_probability_{0}".format(title), show=True)
outfile.write(sft.format_success_msg(success))
if debug:
print "SUMMARY: Success={0}\n".format(success)
return success
def create_report_file(drug_start_timestep, inactivation_times, active_count, inactivations, drug_inactivation_rate, report_name, debug = False):
with open(report_name, "w") as outfile:
success = True
# ks exponential test doesn't work very well with large rate, use chi squared test instead.
# while rate is small ks test for exponential distribution is more sensitive to catch the difference
if drug_inactivation_rate < 0.1:
outfile.write( "Testing inactivation times as draws from exponential distrib with rate {0}. "
"Dataset size = {1}.\n".format( drug_inactivation_rate, len( inactivation_times ) ) )
success = dtk_sft.test_exponential( inactivation_times, drug_inactivation_rate, outfile, integers=True,
roundup=True, round_nearest=False )
if not success:
outfile.write("BAD: ks test for rate {} is False.\n".format(drug_inactivation_rate))
size = len(inactivation_times)
scale = 1.0 / drug_inactivation_rate
dist_exponential_np = numpy.random.exponential(scale, size)
dist_exponential_np = [math.ceil(x) for x in dist_exponential_np]
dtk_sft.plot_data_sorted(inactivation_times, dist_exponential_np,
label1="test times", label2="numpy data",
title="inactivation_times_actual_vs_numpy",
xlabel="data points", ylabel="Inactivation times",
category="inactivation_times", show = True, line = True, overlap=True)
dtk_sft.plot_cdf(inactivation_times, dist_exponential_np,
label1="test times", label2="numpy data",
title="inactivation_times_cdf",
xlabel="days", ylabel="probability",
category="inactivation_times_cdf", show = True)
dtk_sft.plot_probability(inactivation_times, dist_exponential_np,
label1="test times", label2="numpy data",
title="inactivation_times_pdf",
xlabel="days", ylabel="probability",
category="inactivation_times_pdf", show = True)
else:
outfile.write("Testing inactivation count per day with rate {0}. \n".format( drug_inactivation_rate) )
expected_inactivation = []
for t in range( len(inactivations)):
if t < drug_start_timestep :
if inactivations[t] > 0:
success = False
outfile.write("BAD: expected no inactivations on drugs before day {0}, get {1} cases at timestep {2}.\n"
"".format(drug_start_timestep , inactivations[t], t))
elif active_count[t] > 0:
expected_inactivation.append(drug_inactivation_rate * active_count[t])
if len(inactivations) <= len(expected_inactivation) + drug_start_timestep:
test_inactivation_dates = inactivations[drug_start_timestep+1:]
expected_inactivation = expected_inactivation[:len(test_inactivation_dates)]
else:
test_inactivation_dates = inactivations[drug_start_timestep + 1:drug_start_timestep + 1 + len(expected_inactivation)]
#print (len(inactivations), len(test_inactivation_dates), len(expected_inactivation))
#print (test_inactivation_dates, expected_inactivation)
dtk_sft.plot_data(test_inactivation_dates, expected_inactivation,
label1="actual inactivation", label2="expected inactivation",
title="inactivation per day",
xlabel="date after drug start day", ylabel="inactivation per day",
category="inactivation_counts", show=True, line=True, overlap=True, sort=False)
chi_result = dtk_sft.test_multinomial(dist=test_inactivation_dates, proportions=expected_inactivation,
report_file=outfile, prob_flag=False)
if not chi_result:
success = False
outfile.write("BAD: Chi-squared test reuslt is False.\n")
outfile.write(dtk_sft.format_success_msg(success))
if debug:
print(dtk_sft.format_success_msg(success))
return success