def application(output_folder="output", stdout_filename="test.txt",
config_filename="config.json",
insetchart_name="InsetChart.json",
report_name=sft.sft_output_filename,
debug=False):
if debug:
print( "output_folder: " + output_folder )
print( "stdout_filename: " + stdout_filename + "\n" )
print( "config_filename: " + config_filename + "\n" )
print( "insetchart_name: " + insetchart_name + "\n" )
print( "report_name: " + report_name + "\n" )
print( "debug: " + str(debug) + "\n" )
sft.wait_for_done()
param_obj = load_emod_parameters(config_filename)
parsed_data = parse_stdout_file(param_obj, stdout_filename, debug=debug)
inset_days = parse_json_report()
parsed_data.extend([param_obj[KEY_TYPHOID_ACUTE_INFECTIOUSNESS]])
data_to_graph = create_report_file(parsed_data, inset_days,
report_name=sft.sft_output_filename)
sft.plot_data_sorted(data_to_graph[0], data_to_graph[1], label1="Actual", label2="Expected",
title="Contact Dose Response Probability", xlabel="Occurrences",
ylabel="Dose Response",
category='contact_dose_response_probability_plot',
alpha=0.5, overlap=True)
sft.plot_data_sorted(data_to_graph[2], data_to_graph[3], label1="Actual", label2="Expected",
title="Environmental Dose Response Probability", xlabel="Occurrences",
ylabel="Dose Response",
category='environmental_dose_response_probability_plot',
alpha=0.5, overlap=True)
def create_report_file(data, debug=False):
report_name = data[0]
lines = data[1]
tb_cd4_activation_vector = data[2] # this test assumes the vector is constant
latency_data = {}
duration_data = {}
success = True
with open(report_name, "w") as outfile:
if not lines:
outfile.write("BAD: No relevant test data found.\n")
success = False
for line in lines:
if "LifeCourseLatencyTimerUpdate" in line:
ind_id = int(sft.get_val("Individual ", line))
start_time_stamp = int(sft.get_val("time= ", line))
if ind_id in latency_data.keys():
outfile.write("Individual {} incubation timer reset at time {}. Please check. "
"\n".format(ind_id, start_time_stamp))
latency_data[ind_id] = start_time_stamp
elif "TBActivationPresymptomatic" in line:
ind_id = int(sft.get_val("Individual ", line))
end_time_stamp = int(sft.get_val("time= ", line))
if ind_id not in latency_data.keys():
outfile.write("Individual {} went presymptomatic without incubation timer update at time {}. "
"Please check. \n".format(ind_id, end_time_stamp))
else:
duration = end_time_stamp - latency_data.get(ind_id)
duration_data[ind_id] = duration
if debug:
with open("DEBUG_duration_data.json","w") as debug_outfile:
json.dump(duration_data, debug_outfile, indent=4)
durations = list(duration_data.values())
if not sft.test_exponential(durations, tb_cd4_activation_vector[0], outfile, integers=True,
roundup=True, round_nearest=False):
success = False
outfile.write("Data points checked = {}.\n".format(len(duration_data)))
outfile.write("SUMMARY: Success={0}\n".format(success))
# only used for graphing purposes
expected_data = map(math.ceil, np.random.exponential(1/tb_cd4_activation_vector[0], len(duration_data)))
expected_durations = list(expected_data)
sft.plot_data_sorted(durations, expected_durations, label1="Actual", label2="Expected",
title="Recalculated Latency Duration TB then HIV(Sorted)", xlabel="Data Points",
ylabel="Days",
category="tb_activation_and_cd4_tb_first", line = True, overlap=True)
def application(output_folder="output",
stdout_filename="test.txt",
config_filename="config.json",
chart_name="InsetChart.json",
report_name=sft.sft_output_filename,
debug=False):
if debug:
print("output_folder: " + output_folder)
print("stdout_filename: " + stdout_filename + "\n")
print("config_filename: " + config_filename + "\n")
print("chart_name: " + chart_name + "\n")
print("report_name: " + report_name + "\n")
print("debug: " + str(debug) + "\n")
sft.wait_for_done()
param_obj = load_emod_parameters(config_filename)
drug_start_timestep = param_obj[dts.ParamKeys.KEY_DrugStartTime]
sim_start_timestep = param_obj[dts.ConfigKeys.KEY_StartTime]
# Now process log output (probably) and compare to theory (not in this example) or to another report.
duration_of_interest = param_obj[dts.ParamKeys.KEY_DrugRegimenLength]
cum_clears, clear_times, stdout_days = parse_stdout_file(
drug_start_timestep, duration_of_interest, stdout_filename, debug)
inset_days = parse_json_report(sim_start_timestep, debug=debug)
if debug:
print("trying to plot data\n")
sft.plot_data_sorted(
cum_clears,
label1="Cumulative Clearances",
title="Cumulative Clearances over Time from Drugs (HIV+/No ART)",
xlabel="Timestep",
ylabel="Clearances",
show=True)
else:
sft.plot_data(
cum_clears,
label1="Cumulative Clearances",
title="Cumulative Clearances over Time from Drugs (HIV+/No ART)",
xlabel="Timestep",
ylabel="Clearances")
#inset_days = parse_json_report(start_timestep, output_folder, chart_name, debug)
# Now we've ingested all relevant inputs, let's do analysis
if debug:
print("trying to create report file\n")
create_report_file(clear_times, param_obj, report_name, stdout_days,
inset_days, debug)
def create_report_file(data):
report_name = data[0]
lines = data[1]
cd4_strata = data[2]
mod_array = data[3]
actual_data = []
expected_data = []
success = True
epsilon = 0.000002
with open(report_name, "w") as outfile:
if not lines:
outfile.write("BAD: No relevant test data found.\n")
success = False
for line in lines:
cd4_count = float(sft.get_val("CD4count=", line))
cd4_mod_actual = float(sft.get_val("CD4mod=", line))
cd4_mod_expected = tb_cd4_susceptibility_calc(
[mod_array, cd4_strata, cd4_count])
actual_data.append(cd4_mod_actual)
expected_data.append(cd4_mod_expected)
if abs(cd4_mod_actual - cd4_mod_expected) > epsilon:
success = False
outfile.write(
"BAD: At Time: {} for Individual {} with CD4 count {} Expected susceptibility modifier "
"was {}, but actual was {}.\n".format(
sft.get_val("time= ", line),
sft.get_val("Individual ", line), cd4_count,
cd4_mod_expected, cd4_mod_actual))
outfile.write("Data points checked = {} .\n".format(len(lines)))
outfile.write("SUMMARY: Success={0}\n".format(success))
sft.plot_data_sorted(actual_data,
expected_data,
label1="Actual",
label2="Expected",
title="Susceptibility Modifier",
xlabel="Data Points",
ylabel="Modifying Multiplier",
category="tb_susceptibility_and_cd4",
line=True,
overlap=True)
def application(output_folder="output",
stdout_filename="test.txt",
config_filename="config.json",
insetchart_name="InsetChart.json",
report_name=sft.sft_output_filename,
debug=False):
if debug:
print("output_folder: " + output_folder)
print("stdout_filename: " + stdout_filename + "\n")
print("config_filename: " + config_filename + "\n")
print("insetchart_name: " + insetchart_name + "\n")
print("report_name: " + report_name + "\n")
print("debug: " + str(debug) + "\n")
#sft.wait_for_done()
param_obj = load_emod_parameters(config_filename)
parsed_data = parse_stdout_file(debug=debug)
inset_days = parse_json_report()
parsed_data.extend([
param_obj.get(KEY_SIMULATION_DURATION),
param_obj.get(KEY_TYPHOID_EXPOSURE_LAMBDA), sft.sft_output_filename
])
theory = create_report_file(parsed_data)
sft.plot_data_sorted(
parsed_data[0],
theory,
label1="Actual",
label2="Expected",
title=
"Percent of Population Susceptible as they age from 0 to 20, TEL={}"
"".format(parsed_data[3]),
xlabel="Time",
ylabel="Total % Susceptible",
category='immunity_newborns',
alpha=0.5,
overlap=True)
def application(report_file):
sft.wait_for_done()
lines = []
with open("test.txt") as logfile:
for line in logfile:
if re.search("Initializing Typhoid immunity object",
line) and re.search("age=0.000000", line):
lines.append(line)
success = True
actual_newborn_immunity_data = []
expected_newborn_immunity_data = []
with open(sft.sft_output_filename, "w") as report_file:
if not lines:
success = False
report_file.write("BAD: Found no data matching test case.\n")
else:
for line in lines:
immunity = float(sft.get_val(" immunity modifier", line))
actual_newborn_immunity_data.append(immunity)
expected_newborn_immunity_data.append(1)
if immunity != 1.000:
success = False
report_file.write(
"BAD: immunity for newborn={0} instead of 1.0\n".
format(immunity))
report_file.write(sft.format_success_msg(success))
sft.plot_data_sorted(actual_newborn_immunity_data,
expected_newborn_immunity_data,
label1="Actual",
label2="Expected",
title="Newborn Immunity Data",
xlabel="Individuals",
ylabel="Immunity",
category='multi_mode_intervention')
def create_report_file(param_obj, output_dict, report_name, debug):
with open(report_name, "w") as outfile:
config_name = param_obj[KEY_CONFIG_NAME]
outfile.write("Config_name = {}\n".format(config_name))
success = True
cure_rate = param_obj[KEY_CURE_RATE]
simulation_duration = param_obj[KEY_DURATION]
if not len(output_dict):
success = False
outfile.write(sft.sft_no_test_data)
cure_timer = []
actual_timer = []
outfile.write(
"Checking the timer and actual timestep between active and cure:\n"
)
outfile.write(
"checking if the internal timer matches the PreSymptomatic to Cleared duration:\n"
)
for id in output_dict:
cure_time = timer = active_time = None
if KEY_CLEARED in output_dict[id]:
cure_time = output_dict[id][KEY_CLEARED][0]
timer = output_dict[id][KEY_CLEARED][1]
cure_timer.append(timer)
if KEY_SYMPTOMATIC in output_dict[id]:
active_time = output_dict[id][KEY_SYMPTOMATIC][0]
if active_time:
if cure_time: # some individual may not be cleared at the end of the simulation
actual_timer.append(cure_time - active_time)
if cure_time - active_time != math.ceil(timer):
success = False
outfile.write(
"BAD: individual {0} has cure timer = {1} but the actual cure time is {2} (enter "
"symptomatic active state at timestep {3}, cleared at timestep {4}).\n"
.format(id, timer, cure_time - active_time,
active_time, cure_time))
else:
outfile.write(
"Individual {0} moved to symptomatic active at timestep {1} and is not cleared yet at "
"the end of simulation (duration = {2}).\n".format(
id, active_time, simulation_duration))
else:
success = False
outfile.write(
"BAD: individual {0} is cleared before entering active symptomatic state.\n"
.format(id))
if not len(actual_timer):
success = False
outfile.write(
"BAD: There is no recovered individual in this test, please fix the test.\n"
)
outfile.write(
"Result is {0}. # of recovered individual = {1}\n".format(
success, len(actual_timer)))
outfile.write(
"Running ks test for timer and numpy exponential distribution: \n")
size = len(cure_timer)
scale = 1.0 / cure_rate
dist_exponential_np = np.random.exponential(scale, size)
sft.plot_data_sorted(cure_timer,
dist2=np.array(dist_exponential_np),
label1="cure timer",
label2="numpy exponential",
title="exponential rate = {}".format(cure_rate),
xlabel="data point",
ylabel="cure timer",
category='Cure_timer',
show=True,
line=False,
overlap=True)
result = sft.test_exponential(cure_timer,
p1=cure_rate,
report_file=outfile,
integers=False,
roundup=False,
round_nearest=False)
outfile.write(
"ks test result is {0}, exponential rate = {1}, # of data point = {2}.\n"
.format(result, cure_rate, size))
if not result:
success = False
outfile.write(
"BAD: test exponential for cure timer failed with cure rate = {}.\n"
.format(cure_rate))
outfile.write(sft.format_success_msg(success))
if debug:
print("SUMMARY: Success={0}\n".format(success))
return success
def create_report_file(param_obj, output_dict, report_name, debug):
with open(report_name, "w") as outfile:
config_name = param_obj[KEY_CONFIG_NAME]
outfile.write("Config_name = {}\n".format(config_name))
success = True
slow_progressor_rate = param_obj[KEY_SLOW_PROGRESSOR_RATE]
fast_progressor_rate = param_obj[KEY_FAST_PROGRESSOR_RATE]
child_fast_fraction = param_obj[KEY_CHILD_FRACTION]
adult_fast_fraction = param_obj[KEY_ADULT_FRACTION]
progression_multiplier = np.mean(param_obj[KEY_CD4_PROGRESSION_MULTIPLIER])
simulation_duration = param_obj[KEY_DURATION]
if not len(output_dict):
success = False
outfile.write(sft.sft_no_test_data)
outfile.write("checking test conditions: \n")
if child_fast_fraction:
success = False
outfile.write("BAD: expected {0} = 0, got {1} from config.json. "
"Please fix the test.\n".format(KEY_CHILD_FRACTION, child_fast_fraction))
dist_exponential_np_slow = np.random.exponential(1/slow_progressor_rate, 100)
if min(dist_exponential_np_slow) < simulation_duration:
success = False
outfile.write("BAD: expected a small {0} to distinguish fast and slow progress TB, got {1} from config.json. Please "
"fix the test.\n".format(KEY_SLOW_PROGRESSOR_RATE, slow_progressor_rate))
outfile.write("conditional check result is {}.\n".format(success))
actual_timer = []
internal_timer = []
slow_count = 0
outfile.write("collecting the actual timestep between latent and presymptomatic:\n")
outfile.write("checking if the internal timer matches the PreSymptomatic to Cleared duration:\n")
for id in output_dict:
presymptomatic_time = timer = latent_time = None
if KEY_PRESYMPTOMATIC in output_dict[id]:
presymptomatic_time = output_dict[id][KEY_PRESYMPTOMATIC][0]
timer = output_dict[id][KEY_PRESYMPTOMATIC][1]
internal_timer.append(timer)
if KEY_LATENT in output_dict[id]:
latent_time = output_dict[id][KEY_LATENT]
if latent_time:
if presymptomatic_time: # some individual may not move to presymptomatic state at the end of the simulation
actual_timer.append(presymptomatic_time - latent_time)
if presymptomatic_time - latent_time != math.ceil(timer):
success = False
outfile.write("BAD: individual {0} has internal timer = {1} but the actual timer is {2} (enter "
"latent state at timestep {3}, enter presymptomatic active state at "
"timestep {4}).\n".format(id, timer, presymptomatic_time - latent_time,
latent_time, presymptomatic_time))
else:
slow_count += 1
if debug:
outfile.write("Individual {0} moved to latent state at timestep {1} and is not move to "
"presymptomatic active yet at the end of simulation (duration = {2})."
"\n".format(id, latent_time, simulation_duration))
else:
success = False
outfile.write("BAD: individual {0} moved to presymptomatic active state at timerstep {1} before entering "
"latent state.\n".format(id, presymptomatic_time))
if not len(actual_timer):
success = False
outfile.write("BAD: There is no latent to presymptomatic state transition in this test, please fix the test.\n")
outfile.write("Running ks test for latent to presymptomatic internal timer and numpy exponential distribution: \n")
size = len(internal_timer)
scale = 1.0 / fast_progressor_rate
dist_exponential_np = np.random.exponential(scale, size)
sft.plot_data_sorted(internal_timer, dist2=np.array(dist_exponential_np), label1="internal timer",
label2="numpy exponential", title="exponential rate = {}".format(fast_progressor_rate),
xlabel="data point", ylabel="latent to presymptomatic internal timer",
category='latent_to_presymptomatic_internal_timer', show=True, line=False, overlap=True)
result = sft.test_exponential(internal_timer, p1=fast_progressor_rate, report_file=outfile, integers=True, roundup=True,
round_nearest=False)
outfile.write("ks test result is {0}, exponential rate = {1}, # of data point = {2}.\n".format(result, fast_progressor_rate, size))
if not result:
success = False
outfile.write("BAD: test exponential for latent to presymptomatic duration failed with fast_progressor_rate "
"= {}.\n".format(fast_progressor_rate))
else:
outfile.write(
"GOOD: test exponential for latent to presymptomatic duration passed with fast_progressor_rate "
"= {}.\n".format(fast_progressor_rate))
outfile.write("running binomial test with 95% confidence for Fast_Progressor_Fraction_Adult:\n")
result2 = sft.test_binomial_95ci(num_success=len(internal_timer), num_trials=len(internal_timer) + slow_count,
prob=adult_fast_fraction * progression_multiplier, report_file=outfile,
category="Fast_Progressor_Fraction_Adult")
outfile.write("number of slow progressor is {0} and number of fast progressor is {1}.\n".format(slow_count, len(internal_timer)))
if not result2:
success = False
outfile.write("BAD: binomial test for Fast_Progressor_Fraction_Adult = {0} and TB_CD4_Primary_Progression= {1} failed"
".\n".format(adult_fast_fraction, progression_multiplier))
else:
outfile.write("GOOD: binomial test for Fast_Progressor_Fraction_Adult = {0} and TB_CD4_Primary_Progression= {1} passed"
".\n".format(adult_fast_fraction, progression_multiplier))
outfile.write(sft.format_success_msg(success))
if debug:
print( "SUMMARY: Success={0}\n".format(success) )
return success
def create_report_file(param_obj, output_dict, report_name, debug):
with open(report_name, "w") as outfile:
config_name = param_obj[KEY_CONFIG_NAME]
outfile.write("Config_name = {}\n".format(config_name))
success = True
slow_progressor_rate = param_obj[KEY_SLOW_PROGRESSOR_RATE]
simulation_duration = param_obj[KEY_DURATION]
if not len(output_dict):
success = False
outfile.write(sft.sft_no_test_data)
actual_timer = []
internal_timer = []
outfile.write("collecting the actual timestep between latent and presymptomatic:\n")
outfile.write("checking if the internal timer matches the PreSymptomatic to Cleared duration:\n")
for id in output_dict:
presymptomatic_time = timer = latent_time = None
if KEY_PRESYMPTOMATIC in output_dict[id]:
presymptomatic_time = output_dict[id][KEY_PRESYMPTOMATIC][0]
timer = output_dict[id][KEY_PRESYMPTOMATIC][1]
internal_timer.append(timer)
if KEY_LATENT in output_dict[id]:
latent_time = output_dict[id][KEY_LATENT]
if latent_time:
if presymptomatic_time: # some individual may not move to presymptomatic state at the end of the simulation
actual_timer.append(presymptomatic_time - latent_time)
if presymptomatic_time - latent_time != math.ceil(timer):
success = False
outfile.write("BAD: individual {0} has internal timer = {1} but the actual timer is {2} (enter "
"latent state at timestep {3}, enter presymptomatic active state at "
"timestep {4}).\n".format(id, timer, presymptomatic_time - latent_time,
latent_time, presymptomatic_time))
else:
outfile.write("Individual {0} moved to latent state at timestep {1} and is not move to "
"presymptomatic active yet at the end of simulation (duration = {2})."
"\n".format(id, latent_time, simulation_duration))
else:
success = False
outfile.write("BAD: individual {0} moved to presymptomatic active state at timerstep {1} before entering "
"latent state.\n".format(id, presymptomatic_time))
if not len(actual_timer):
success = False
outfile.write("BAD: There is no latent to presymptomatic state transition in this test, please fix the test.\n")
outfile.write("Running ks test for latent to presymptomatic internal timer and numpy exponential distribution: \n")
size = len(internal_timer)
scale = 1.0 / slow_progressor_rate
dist_exponential_np = np.random.exponential(scale, size)
sft.plot_data_sorted(internal_timer, dist2=np.array(dist_exponential_np), label1="latent to presymptomatic internal timer",
label2="numpy exponential", title="exponential rate = {}".format(slow_progressor_rate),
xlabel="data point", ylabel="latent to presymptomatic internal timer",
category='latent_to_presymptomatic_internal_timer', show=True, line=False, overlap=True)
result = sft.test_exponential(internal_timer, p1=slow_progressor_rate, report_file=outfile, integers=True, roundup=True,
round_nearest=False)
outfile.write("ks test result is {0}, exponential rate = {1}, # of data point = {2}.\n".format(result, slow_progressor_rate, size))
if not result:
success = False
outfile.write("BAD: test exponential for latent to presymptomatic internal timer failed with slow_progressor_rate "
"= {}.\n".format(slow_progressor_rate))
else:
outfile.write("GOOD: test exponential for latent to presymptomatic internal timer passed with slow_progressor_rate "
"= {}.\n".format(slow_progressor_rate))
outfile.write(sft.format_success_msg(success))
if debug:
print( "SUMMARY: Success={0}\n".format(success) )
return success
def create_report_file(param_obj, multipliers, infectiousness, report_name,
debug):
with open(report_name, "w") as outfile:
success = True
if not multipliers:
outfile.write(sft.sft_no_test_data)
sigma = param_obj[Param_keys.LOGNORMAL_SCALE]
base_infectivity = param_obj[Param_keys.BASE_INFECTIVITY]
if sigma > 0:
mu = -sigma**2 / 2.0
# test log_normal distribution
success = sft.test_lognorm(multipliers,
mu=mu,
sigma=sigma,
report_file=outfile,
round=False)
# test mean_l = 1
mean_l = np.mean(multipliers)
mean_infectiousness = np.mean(infectiousness)
outfile.write(
"mean of the multipliers is {}, expected 1.0.\n".format(
mean_l))
outfile.write(
"mean of the Infectiousness is {0}, while base infectivity is {1}.\n"
.format(mean_infectiousness, base_infectivity))
tolerance = 3e-2
if math.fabs(mean_l - 1.0) > tolerance:
outfile.write(
"BAD: mean of the multipliers is {}, expected 1.0.\n".
format(mean_l))
success = False
# plotting
size = len(multipliers)
outfile.write("size is {}\n".format(size))
scale = math.exp(mu)
dist_lognormal = stats.lognorm.rvs(sigma, 0, scale, size)
sft.plot_data_sorted(
multipliers,
dist_lognormal,
label1="Emod",
label2="Scipy",
ylabel="Multiplier",
xlabel="data point",
category="Emod_vs_Scipy",
title="Emod_vs_Scipy, sigma = {}".format(sigma),
show=True)
sft.plot_probability(
multipliers,
dist_lognormal,
precision=1,
label1="Emod",
label2="Scipy",
category="Probability_mass_function_Emod_vs_Scipy",
title="Emod_vs_Scipy, sigma = {}".format(sigma),
show=True)
sft.plot_cdf(multipliers,
dist_lognormal,
label1="Emod",
label2="Scipy",
category="cdf",
title="cdf, sigma = {}".format(sigma),
show=True,
line=False)
if debug:
with open("scipy_data.txt", "w") as file:
for n in sorted(dist_lognormal):
file.write(str(n) + "\n")
with open("emod_data.txt", "w") as file:
for n in sorted(multipliers):
file.write(str(n) + "\n")
else:
# sigma = 0, this feature is disabled
for multiplier in multipliers:
if multiplier != 1.0:
success = False
outfile.write(
"BAD: multiplier is {0} when {1} set to {2}, expected 1.0.\n"
.format(multiplier, Param_keys.LOGNORMAL_SCALE, sigma))
# plotting
sft.plot_data_sorted(multipliers,
label1="Multiplier",
label2="NA",
category="Multiplier",
title="Multiplier_Sigma={}".format(sigma),
ylabel="Multiplier",
xlabel="data point",
show=True)
sft.plot_data_sorted(
infectiousness,
label1="Infectiousness",
label2="NA",
category="Infectiousness",
title="Infectiousness_Sigma={0}_BaseInfectivity={1}".format(
sigma, base_infectivity),
ylabel="Infectiousness",
xlabel="data point",
show=True)
outfile.write(sft.format_success_msg(success))
if debug:
print("SUMMARY: Success={0}\n".format(success))
return success
def create_report_file(drug_start_timestep,
inactivation_times,
active_count,
inactivations,
drug_inactivation_rate,
report_name,
debug=False):
with open(report_name, "w") as outfile:
success = True
# ks exponential test doesn't work very well with large rate, use chi squared test instead.
# while rate is small ks test for exponential distribution is more sensitive to catch the difference
if drug_inactivation_rate < 0.1:
outfile.write(
"Testing inactivation times as draws from exponential distrib with rate {0}. "
"Dataset size = {1}.\n".format(drug_inactivation_rate,
len(inactivation_times)))
success = sft.test_exponential(inactivation_times,
drug_inactivation_rate,
outfile,
integers=True,
roundup=True,
round_nearest=False)
if not success:
outfile.write("BAD: ks test for rate {} is False.\n".format(
drug_inactivation_rate))
size = len(inactivation_times)
scale = 1.0 / drug_inactivation_rate
dist_exponential_np = numpy.random.exponential(scale, size)
dist_exponential_np = [math.ceil(x) for x in dist_exponential_np]
sft.plot_data_sorted(inactivation_times,
dist_exponential_np,
label1="test times",
label2="numpy data",
title="inactivation_times_actual_vs_numpy",
xlabel="data points",
ylabel="Inactivation times",
category="inactivation_times",
show=True,
line=True,
overlap=True)
sft.plot_cdf(inactivation_times,
dist_exponential_np,
label1="test times",
label2="numpy data",
title="inactivation_times_cdf",
xlabel="days",
ylabel="probability",
category="inactivation_times_cdf",
show=True)
sft.plot_probability(inactivation_times,
dist_exponential_np,
label1="test times",
label2="numpy data",
title="inactivation_times_pdf",
xlabel="days",
ylabel="probability",
category="inactivation_times_pdf",
show=True)
else:
outfile.write(
"Testing inactivation count per day with rate {0}. \n".format(
drug_inactivation_rate))
expected_inactivation = []
for t in range(len(inactivations)):
if t < drug_start_timestep:
if inactivations[t] > 0:
success = False
outfile.write(
"BAD: expected no inactivations on drugs before day {0}, get {1} cases at timestep {2}.\n"
"".format(drug_start_timestep, inactivations[t],
t))
elif active_count[t] > 0:
expected_inactivation.append(drug_inactivation_rate *
active_count[t])
if len(inactivations
) <= len(expected_inactivation) + drug_start_timestep:
test_inactivation_dates = inactivations[drug_start_timestep +
1:]
expected_inactivation = expected_inactivation[:len(
test_inactivation_dates)]
else:
test_inactivation_dates = inactivations[
drug_start_timestep + 1:drug_start_timestep + 1 +
len(expected_inactivation)]
#print (len(inactivations), len(test_inactivation_dates), len(expected_inactivation))
#print (test_inactivation_dates, expected_inactivation)
sft.plot_data(test_inactivation_dates,
expected_inactivation,
label1="actual inactivation",
label2="expected inactivation",
title="inactivation per day",
xlabel="date after drug start day",
ylabel="inactivation per day",
category="inactivation_counts",
show=True,
line=True,
overlap=True,
sort=False)
chi_result = sft.test_multinomial(
dist=test_inactivation_dates,
proportions=expected_inactivation,
report_file=outfile,
prob_flag=False)
if not chi_result:
success = False
outfile.write("BAD: Chi-squared test reuslt is False.\n")
outfile.write(sft.format_success_msg(success))
if debug:
print(sft.format_success_msg(success))
return success
def application(report_file, debug=False):
sft.wait_for_done()
"""
Parse this line from test.txt:
00:00:00 [0] [V] [IndividualTyphoid] amplification calculated as 0.997059: day of year=1, start=360.000000,
end=365.000000, ramp_up=30.000000, ramp_down=170.000000, cutoff=160.000000.
00:00:00 [0] [V] [IndividualTyphoid] Exposing individual 2 age 8582.488281 on route 'environment': prob=0.000000,
infects=0.000008, immunity=1.000000, num_exposures=0, exposure=0.997059, environment=1.000000, iv_mult=1.000000.
"""
# print( "Post-processing: " + report_file )
# get params from config.json
cdj = json.loads(open("config.json").read())["parameters"]
start_time = cdj["Start_Time"]
lines = []
timestep = start_time
count = 0
with open("test.txt") as logfile:
for line in logfile:
if re.search("Update\(\): Time:", line):
# calculate time step
timestep += 1
line = "TimeStep: " + str(timestep) + " " + line
lines.append(line)
if re.search("amplification calculated as", line):
count += 1
line = "TimeStep: " + str(timestep) + " " + line
lines.append(line)
if re.search("Exposing", line) and re.search("environment", line):
line = "TimeStep: " + str(timestep) + " " + line
lines.append(line)
# more params from config file
rud = cdj["Environmental_Ramp_Up_Duration"]
rdd = cdj["Environmental_Ramp_Down_Duration"]
ecd = cdj["Environmental_Cutoff_Days"]
eps = cdj["Environmental_Peak_Start"]
peak_duration = 365 - rud - rdd - ecd
# for eps > 365
peak_starttime = eps % 365
peak_endtime = peak_starttime + peak_duration
cutoff_starttime = peak_starttime + peak_duration + rdd
success = True
amp = []
environmental_amp = []
with open(sft.sft_output_filename, "w") as report_file:
if count == 0:
success = False
report_file.write("Found no data matching test case.\n")
elif peak_duration < 0:
success = False
report_file.write("BAD: Environmental peak duration should be larger or equal to 0, the actual value is {}."
"\n The ramp durations and cutoff days need to follow a rule where: ramp_up_duration + "
"ramp_down_duration + cutoff_days < 365.\n".format(peak_duration))
else:
# adjust the times so that the ramp up starts at time 0, which means the cut off ends at time 0 too.
adjust_time = peak_starttime - rud
peak_starttime -= adjust_time
peak_endtime -= adjust_time
cutoff_starttime -= adjust_time
for line in lines:
if re.search("Update\(\): Time:", line):
TimeStep = int(sft.get_val("TimeStep: ", line))
TimeStep -= adjust_time
day_in_year = TimeStep % 365
elif re.search("amplification calculated as", line):
amplification = float(sft.get_val("amplification calculated as ", line))
# Environment Ramp Up
if day_in_year < peak_starttime:
environmental_amplification = day_in_year/float(rud)
# Environment peak
elif peak_starttime <= day_in_year <= peak_endtime:
environmental_amplification = 1
# Environment Ramp Down
elif peak_endtime < day_in_year < cutoff_starttime:
environmental_amplification = (cutoff_starttime - day_in_year) / float(rdd)
# Environment cutoff
elif day_in_year >= cutoff_starttime:
environmental_amplification = 0
if math.fabs(amplification - environmental_amplification) > 5e-2:
success = False
TimeStep = int(sft.get_val("TimeStep: ", line))
report_file.write("BAD: at time {0}, day of year = {1}, the environmental amplification is {2},"
" expected {3}.\n".format(TimeStep, day_in_year+adjust_time, amplification,
environmental_amplification))
amp.append(amplification)
environmental_amp.append(environmental_amplification)
elif re.search("Exposing", line):
ind_id = sft.get_val("individual ", line)
environment = float(sft.get_val("environment=", line))
exposure = float(sft.get_val("exposure=", line))
expected_exposure = environment * amplification
if math.fabs(expected_exposure - exposure) > 1e-2:
success = False
TimeStep = int(sft.get_val("TimeStep: ", line))
report_file.write("BAD: at time {0}, day of year = {1}, the amount of environmental contagion "
"that individual (2) is exposed is {3}, expected {4}"
".\n".format(TimeStep, day_in_year + adjust_time, ind_id,
exposure, expected_exposure))
sft.plot_data_sorted(amp, environmental_amp,
label1="Actual Seasonal Attenuation",
label2="Expected Seasonal Attenuation",
title="Seasonal Attenuation Peak Equals RUD plus CD plus RDD", xlabel="Time",
ylabel="Attenuation",
category='seasonal_attenuation_peakstartequalsRUDplusCDplusRDD')
report_file.write(sft.format_success_msg(success))
def create_report_file(param_obj, output_dict, report_name, debug):
with open(report_name, "w") as outfile:
config_name = param_obj[KEY_CONFIG_NAME]
outfile.write("Config_name = {}\n".format(config_name))
success = True
fast_progressor_rate = param_obj[KEY_FAST_PROGRESSOR_RATE]
latent_cure_rate = param_obj[KEY_LATENT_CURE_RATE]
child_fast_fraction = param_obj[KEY_CHILD_FRACTION]
adult_fast_fraction = param_obj[KEY_ADULT_FRACTION]
simulation_duration = param_obj[KEY_DURATION]
if not len(output_dict):
success = False
outfile.write(sft.sft_no_test_data)
outfile.write("checking test conditions: \n")
if not child_fast_fraction or not adult_fast_fraction:
success = False
outfile.write("BAD: expected {0} and {1} = 1, got {2} and {3} from config.json. "
"Please fix the test.\n".format(KEY_CHILD_FRACTION, KEY_ADULT_FRACTION, child_fast_fraction, adult_fast_fraction))
dist_exponential_np_fast = np.random.exponential(1 / fast_progressor_rate, 100)
if min(dist_exponential_np_fast) < simulation_duration:
success = False
outfile.write("BAD: expected a small {0} to avoid moving individual to active disease state, got {1} from config.json. Please "
"fix the test.\n".format(KEY_FAST_PROGRESSOR_RATE, fast_progressor_rate))
outfile.write("conditional check result is {}.\n".format(success))
actual_timer = []
outfile.write("collecting the actual timestep between latent and cleared:\n")
for id in output_dict:
cleared_time = presymptomatic_time = latent_time = None
if KEY_CLEARED in output_dict[id]:
cleared_time = output_dict[id][KEY_CLEARED]
if KEY_LATENT in output_dict[id]:
latent_time = output_dict[id][KEY_LATENT]
if KEY_PRESYMPTOMATIC in output_dict[id]:
presymptomatic_time = output_dict[id][KEY_PRESYMPTOMATIC]
if latent_time:
if cleared_time: # some individual may not move to cleared state at the end of the simulation
actual_timer.append(cleared_time - latent_time)
else:
outfile.write("Individual {0} moved to latent state at timestep {1} and is not cleared yet at the "
"end of simulation (duration = {2})."
"\n".format(id, latent_time, simulation_duration))
else:
success = False
outfile.write("BAD: individual {0} moved to cleared state at timerstep {1} before entering "
"latent state.\n".format(id, cleared_time))
if presymptomatic_time:
success = False
outfile.write("BAD: individual {0} moved to presymptomatic at timestep {1}, expected no active disease"
" in this simulation, please double check the config.\n".format(id, presymptomatic_time))
if not len(actual_timer):
success = False
outfile.write("BAD: There is no latent to cleared transition in this test, please fix the test.\n")
outfile.write("Running ks test for latent to cleared duration and numpy exponential distribution: \n")
size = len(actual_timer)
scale = 1.0 / latent_cure_rate
dist_exponential_np = np.random.exponential(scale, size)
sft.plot_data_sorted(actual_timer, dist2=np.array(dist_exponential_np), label1="latent to cleared duration",
label2="numpy exponential", title="exponential rate = {}".format(latent_cure_rate),
xlabel="data point", ylabel="latent to cleared duration",
category='latent_to_cleared_duration', show=True, line=False, overlap=True)
result = sft.test_exponential(actual_timer, p1=latent_cure_rate, report_file=outfile, integers=True, roundup=True,
round_nearest=False)
outfile.write("ks test result is {0}, exponential rate = {1}, # of data point = {2}.\n".format(result, latent_cure_rate, size))
if not result:
success = False
outfile.write("BAD: test exponential for latent to cleared duration failed with {0} "
"= {1}.\n".format(KEY_LATENT_CURE_RATE, latent_cure_rate))
else:
outfile.write("GOOD: test exponential for latent to cleared duration passed with {0} "
"= {1}.\n".format(KEY_LATENT_CURE_RATE, latent_cure_rate))
outfile.write(sft.format_success_msg(success))
if debug:
print( "SUMMARY: Success={0}\n".format(success) )
return success
def create_report_file(param_obj, multipliers, infectiousness, report_name, debug):
with open(report_name, "w") as outfile:
success = True
if not multipliers:
outfile.write(sft.sft_no_test_data)
sigma = param_obj[Param_keys.LOGNORMAL_SCALE]
base_infectivity = param_obj[Param_keys.BASE_INFECTIVITY]
if sigma > 0:
mu = - sigma**2 / 2.0
# test log_normal distribution
success = sft.test_lognorm(multipliers,mu=mu, sigma=sigma,report_file=outfile,round = False)
# test mean_l = 1
mean_l = np.mean(multipliers)
mean_infectiousness = np.mean(infectiousness)
outfile.write("mean of the multipliers is {}, expected 1.0.\n".format(mean_l))
outfile.write("mean of the Infectiousness is {0}, while base infectivity is {1}.\n".format(mean_infectiousness,
base_infectivity))
tolerance = 3e-2
if math.fabs(mean_l - 1.0) > tolerance:
outfile.write("BAD: mean of the multipliers is {}, expected 1.0.\n".format(mean_l))
success = False
# plotting
size = len(multipliers)
outfile.write("size is {}\n".format(size))
scale = math.exp(mu)
dist_lognormal = stats.lognorm.rvs(sigma, 0, scale, size)
sft.plot_data_sorted(multipliers, dist_lognormal,
label1="Emod", label2="Scipy",
ylabel="Multiplier", xlabel="data point",
category="Emod_vs_Scipy",
title="Emod_vs_Scipy, sigma = {}".format(sigma),
show=True)
sft.plot_probability(multipliers, dist_lognormal,
precision=1, label1="Emod", label2="Scipy",
category="Probability_mass_function_Emod_vs_Scipy",
title="Emod_vs_Scipy, sigma = {}".format(sigma),
show=True)
sft.plot_cdf(multipliers,dist_lognormal,label1="Emod", label2="Scipy",
category="cdf",
title="cdf, sigma = {}".format(sigma),
show=True, line = False)
if debug:
with open("scipy_data.txt", "w") as file:
for n in sorted(dist_lognormal):
file.write(str(n) + "\n")
with open("emod_data.txt", "w") as file:
for n in sorted(multipliers):
file.write(str(n) + "\n")
else:
# sigma = 0, this feature is disabled
for multiplier in multipliers:
if multiplier != 1.0:
success = False
outfile.write("BAD: multiplier is {0} when {1} set to {2}, expected 1.0.\n".format(multiplier, Param_keys.LOGNORMAL_SCALE, sigma))
# plotting
sft.plot_data_sorted(multipliers, label1="Multiplier", label2="NA",
category="Multiplier", title="Multiplier_Sigma={}".format(sigma),
ylabel="Multiplier", xlabel="data point",
show=True)
sft.plot_data_sorted(infectiousness, label1="Infectiousness",
label2="NA",category="Infectiousness",
title="Infectiousness_Sigma={0}_BaseInfectivity={1}".format(sigma,base_infectivity),
ylabel="Infectiousness",xlabel="data point",
show=True)
outfile.write(sft.format_success_msg(success))
if debug:
print( "SUMMARY: Success={0}\n".format(success) )
return success
def create_report_file(drug_start_timestep, disease_deaths, cum_deaths, deaths, infected_individuals, death_times, drug_mortality_rate_HIV, report_name ):
with open(report_name, "w") as outfile:
success = True
length = len(cum_deaths)
if sum(disease_deaths)==0 or sum(cum_deaths)==0 or len(death_times)==0:
success = False
outfile.write(sft.no_test_data)
for x in range(length):
if disease_deaths[x] != cum_deaths[x]:
success = False
outfile.write("BAD: at timestep {0}, disease deaths is {1} in InsetChart.json and {2} in stdout.txt.\n".format(x+1, disease_deaths[x], cum_deaths[x]))
# ks exponential test doesn't work very well with large rate, use chi squared test instead
# while rate is small ks test for exponential distribution is more sensitive to catch the difference
if drug_mortality_rate_HIV < 0.1:
outfile.write("Testing death times as draws from exponential distrib with rate {0}. "
"Dataset size = {1}.\n".format(drug_mortality_rate_HIV, len(death_times)))
ks_result = sft.test_exponential( death_times, drug_mortality_rate_HIV, report_file = outfile,
integers=True, roundup=True, round_nearest=False )
if not ks_result:
success = False
outfile.write("BAD: ks test reuslt is False.\n")
size = len(death_times)
scale = 1.0 / drug_mortality_rate_HIV
dist_exponential_np = numpy.random.exponential(scale, size)
dist_exponential_np = [math.ceil(x) for x in dist_exponential_np]
sft.plot_data_sorted(death_times, dist_exponential_np,
label1="death times", label2="numpy data",
title="death_times_actual_vs_numpy",
xlabel="data points", ylabel="death times",
category="death_times", show=True, line = True, overlap=True)
sft.plot_cdf(death_times, dist_exponential_np,
label1="death times", label2="numpy data",
title="death_times_cdf",
xlabel="days", ylabel="probability",
category="death_times_cdf", show=True)
else:
outfile.write("Testing death count per day with rate {0}. \n".format(drug_mortality_rate_HIV))
expected_mortality = []
for t in range( len(deaths)):
if t < drug_start_timestep + 1:
if deaths[t] > 0:
success = False
outfile.write("BAD: expected no disease death on drugs before day {0}, get {1} cases at timestep {2}.\n"
"".format(drug_start_timestep + 1, deaths[t], t))
elif infected_individuals[t] > 0:
expected_mortality.append(drug_mortality_rate_HIV * infected_individuals[t])
expected_mortality.pop(0) # the Infected is off by one day
test_death_dates = deaths[drug_start_timestep + 1:drug_start_timestep + 1 + len(expected_mortality)]
sft.plot_data(test_death_dates, expected_mortality,
label1="actual death", label2="expected death",
title="death per day",
xlabel="date after drug start day", ylabel="death per day",
category="death_counts", show=True, line=True, overlap=True, sort=False)
chi_result = sft.test_multinomial(dist=test_death_dates, proportions=expected_mortality,
report_file=outfile, prob_flag=False)
if not chi_result:
success = False
outfile.write("BAD: Chi-squared test reuslt is False.\n")
outfile.write(sft.format_success_msg(success))
return success
def application(do_not_use):
report_file = ""
debug = True
if not report_file:
report_file = "test.txt"
sft.wait_for_done()
# pdb.set_trace()
# print( "Post-processing: " + report_file )
isj = json.loads(open("output/InsetChart.json").read())["Channels"]
chan_title = "New Infections By Route (ENVIRONMENT)"
env_new_infections = isj[chan_title]["Data"]
regex0 = "Risk:Target_Demographic"
regex1 = "Set current_dose_attenuation_environment"
success = False
start_time = datetime.datetime.now()
max_time = datetime.timedelta(seconds=600)
elapsed_time = datetime.timedelta(seconds=0)
while not (success or elapsed_time > max_time):
env_attenuation = []
filtered_lines = []
with open(report_file) as logfile:
for line in logfile:
if regex0 in line and regex1 in line:
env_attenuation.append(float(line.split()[8].rstrip(',')))
if debug:
filtered_lines.append(line)
if len(env_attenuation) + 1 >= len(env_new_infections):
success = True
else:
time.sleep(10)
elapsed_time = datetime.datetime.now() - start_time
if debug:
with open("filtered_lines.txt", "w") as outfile:
outfile.write("First filtered line split: \n")
splits = filtered_lines[0].split()
for i in range(0, len(splits)):
outfile.write("\t " + str(i) + " " + splits[i] + " \n")
outfile.write("Filtered Lines: \n")
for line in filtered_lines:
outfile.write(line)
success = True
with open(sft.sft_output_filename, "w") as report_file:
if not env_attenuation:
# make sure we have found the correct debug data
success = False
report_file.write(
"BAD: Couldn't find current_dose_attenuation_environment.\n")
if len(env_attenuation) + 1 != len(env_new_infections):
success = False
err_template = "BAD: Length difference between Env attenuation: {0} env_new_infections: {1}\n"
report_file.write(
err_template.format(len(env_attenuation),
len(env_new_infections)))
else:
for i in range(0, len(env_new_infections) - 1):
# when attenuation is 0, new infections should be at 0.
if env_attenuation[i] == 0 and env_new_infections[i + 1] != 0:
success = False
err_template = "BAD: At time {0}: the {1} is {2} from Stdout, " + \
"while the new infections by route is {3} from InsetChart.json.\n"
report_file.write(
err_template.format(i + 1, regex1, env_attenuation[i],
env_new_infections[i + 1]))
report_file.write(sft.format_success_msg(success))
# mapping all the large contagion #s to 1s for better viewing of attenuation data
for i in range(0, len(env_new_infections)):
if env_new_infections[i] > 1:
env_new_infections[i] = 1
# plotting more interesting subsection
sft.plot_data_sorted(
env_attenuation[0:1200],
env_new_infections[1:1201],
label1="Attenuation",
label2="New Infections_Env",
title=
"Vaccine Linear Dose Environmental\n(when attenuation is 0, new infections should be 0)",
xlabel="Time",
ylabel="",
category='vaccine_linear_dose_environmental',
overlap=False,
alpha=0.5)
def application(report_file, debug=False):
# pdb.set_trace()
# print( "Post-processing: " + report_file )
sft.wait_for_done()
cdj = json.loads(open("config.json").read())["parameters"]
start_time = cdj["Start_Time"]
isj = json.loads(open(os.path.join("output",
"InsetChart.json")).read())["Channels"]
nocc = isj["Number of Chronic Carriers"]["Data"]
count = 0
sum_count = 0
counts = []
sum_counts = []
with open("test.txt") as logfile:
for line in logfile:
if re.search("Update\(\): Time:", line):
counts.append(count)
sum_count += count
sum_counts.append(sum_count)
count = 0
if re.search("just went chronic", line):
count += 1
success = True
with open(sft.sft_output_filename, "w") as report_file:
if not sum_count:
success = False
report_file.write(
"BAD: Found no Chronic Infections in test case.\n")
else:
pre_sum_count_icj = 0
debug_count = []
# #2890
# for i in range(1, len(counts)):
# timestep = start_time + i
# count_log = counts[i]
# cur_sum_count_icj = nocc[i - 1]
for i in range(0, len(counts)):
timestep = start_time + i
count_log = counts[i]
cur_sum_count_icj = nocc[i]
if cur_sum_count_icj != count_log + pre_sum_count_icj:
success = False
report_file.write(
"BAD: Total Chronic infections is {0} from InsetChart.json at time {1}, it's {2} "
"at previous time step, expected {3} more based on Stdout log."
"\n".format(cur_sum_count_icj, timestep,
pre_sum_count_icj, count_log))
debug_count.append(count_log + pre_sum_count_icj)
pre_sum_count_icj = cur_sum_count_icj
sft.plot_data_sorted(debug_count,
nocc,
label1="Total Chronic from stdout",
label2="Total Chronic from InsertChart",
title="Chronic Infections (InsetChart Test)",
xlabel="Time",
ylabel="Total Number of Chronic Infections",
category='report_inset_chart_total_chronic',
alpha=0.5,
overlap=True)
report_file.write(sft.format_success_msg(success))
def create_report_file(param_obj, output_dict, report_name, debug):
with open(report_name, "w") as outfile:
config_name = param_obj[KEY_CONFIG_NAME]
outfile.write("Config_name = {}\n".format(config_name))
success = True
slow_progressor_rate = param_obj[KEY_SLOW_PROGRESSOR_RATE]
latent_cure_rate = param_obj[KEY_LATENT_CURE_RATE]
presymptomatic_cure_rate = param_obj[KEY_PRESYMPTOMATIC_CURE_RATE]
presymptomatic_rate = param_obj[KEY_PRESYMPTOMATIC_RATE]
base_infectivity = param_obj[KEY_BASE_INFECTIVITY]
simulation_duration = param_obj[KEY_DURATION]
if not len(output_dict):
success = False
outfile.write(sft.sft_no_test_data)
outfile.write("checking test conditions: \n")
dist_exponential_np_slow = np.random.exponential(
1 / slow_progressor_rate, 100)
if min(dist_exponential_np_slow) < simulation_duration:
success = False
outfile.write(
"BAD: expected a small {0} to avoid moving individual to active disease state, got {1} from config.json. Please "
"fix the test.\n".format(KEY_SLOW_PROGRESSOR_RATE,
slow_progressor_rate))
dist_exponential_np_latent_cure = np.random.exponential(
1 / latent_cure_rate, 100)
if min(dist_exponential_np_latent_cure) < simulation_duration:
success = False
outfile.write(
"BAD: expected a small {0} to avoid Latent to Cleared state transition(all Latent state will progress to "
"PreSymptomatic), got {1} from config.json. Please fix the test.\n"
.format(KEY_LATENT_CURE_RATE, latent_cure_rate))
dist_exponential_np_presymptomatic = np.random.exponential(
1 / presymptomatic_rate, 100)
if min(dist_exponential_np_presymptomatic) < simulation_duration:
success = False
outfile.write(
"BAD: expected a small {0} to avoid PreSymptomatic to Symptomatic state transition(all PreSymptomatic "
"state will progress to Cleared), got {1} from config.json. Please fix the test.\n"
.format(KEY_PRESYMPTOMATIC_RATE, presymptomatic_rate))
if base_infectivity:
success = False
outfile.write(
"BAD: expected {0} = 0 to look only at progression, got {1} from config.json. Please fix"
"the test.\n".format(KEY_BASE_INFECTIVITY, base_infectivity))
outfile.write("conditional check result is {}.\n".format(success))
actual_timer = []
internal_timer = []
outfile.write(
"collecting the actual timestep between PreSymptomatic and Cleared:\n"
)
outfile.write(
"checking if the internal timer matches the PreSymptomatic to Cleared duration:\n"
)
for id in output_dict:
cleared_time = presymptomatic_time = timer = None
if KEY_CLEARED in output_dict[id]:
cleared_time = output_dict[id][KEY_CLEARED][0]
timer = output_dict[id][KEY_CLEARED][1]
internal_timer.append(timer)
if KEY_PRESYMPTOMATIC in output_dict[id]:
presymptomatic_time = output_dict[id][KEY_PRESYMPTOMATIC][0]
if presymptomatic_time:
if cleared_time: # some individual may not move to cleared state at the end of the simulation
actual_timer.append(cleared_time - presymptomatic_time)
if cleared_time - presymptomatic_time != math.ceil(timer):
success = False
outfile.write(
"BAD: individual {0} has internal timer = {1} but the actual timer is {2} (enter "
"PreSymptomatic state at timestep {3}, moved to Cleared state at "
"timestep {4}).\n".format(
id, timer, cleared_time - presymptomatic_time,
presymptomatic_time, cleared_time))
else:
outfile.write(
"Individual {0} moved to PreSymptomatic state at timestep {1} and is not cleared yet at the "
"end of simulation (duration = {2})."
"\n".format(id, presymptomatic_time,
simulation_duration))
else:
success = False
outfile.write(
"BAD: individual {0} moved to cleared state at timerstep {1} before entering "
"PreSymptomatic state.\n".format(id, cleared_time))
if not len(actual_timer):
success = False
outfile.write(
"BAD: There is no PreSymptomatic to cleared transition in this test, please fix the test.\n"
)
outfile.write(
"Running ks test for PreSymptomatic to cleared internal timer and numpy exponential distribution: \n"
)
size = len(internal_timer)
scale = 1.0 / presymptomatic_cure_rate
dist_exponential_np = np.random.exponential(scale, size)
sft.plot_data_sorted(
internal_timer,
dist2=np.array(dist_exponential_np),
label1="PreSymptomatic to cleared duration",
label2="numpy exponential",
title="exponential rate = {}".format(presymptomatic_cure_rate),
xlabel="data point",
ylabel="PreSymptomatic to cleared duration",
category='PreSymptomatic_to_cleared_duration',
show=True,
line=True,
overlap=True)
result = sft.test_exponential(internal_timer,
p1=presymptomatic_cure_rate,
report_file=outfile,
integers=False,
roundup=False,
round_nearest=False)
outfile.write(
"ks test result is {0}, exponential rate = {1}, # of data point = {2}.\n"
.format(result, presymptomatic_cure_rate, size))
if not result:
success = False
outfile.write(
"BAD: test exponential for PreSymptomatic to cleared duration failed with {0} "
"= {1}.\n".format(KEY_PRESYMPTOMATIC_CURE_RATE,
presymptomatic_cure_rate))
else:
outfile.write(
"GOOD: test exponential for PreSymptomatic to cleared duration passed with {0} "
"= {1}.\n".format(KEY_PRESYMPTOMATIC_CURE_RATE,
presymptomatic_cure_rate))
outfile.write(sft.format_success_msg(success))
if debug:
print("SUMMARY: Success={0}\n".format(success))
return success
def application(report_file):
sft.wait_for_done()
# print( "Post-processing: " + report_file )
cdj = json.loads(open("config.json").read())["parameters"]
start_time = cdj["Start_Time"]
timestep = start_time
lines = []
count_chronic = 0
count_recovered = 0
count_chronic_daily = []
count_recovered_daily = []
with open(sft.sft_test_filename) as logfile:
for line in logfile:
if re.search("Update\(\): Time:", line):
# calculate time step
timestep += 1
count_chronic_daily.append(count_chronic)
count_recovered_daily.append(count_recovered)
count_chronic = 0
count_recovered = 0
if re.search("just went chronic", line):
# append time step and all Infection stage transition to list
line = "TimeStep: "+str(timestep) + " " + line
lines.append(line)
count_chronic += 1
if re.search("just recovered", line):
# append time step and all Infection stage transition to list
line = "TimeStep: " + str(timestep) + " " + line
lines.append(line)
count_recovered += 1
success = True
with open(sft.sft_output_filename, "w") as report_file:
if len(lines) == 0:
success = False
report_file.write("Found no data matching test case.\n")
else:
for line in lines:
age = float(sft.get_val(" age ", line))
sex = "female" if (re.search("sex 1", line) or re.search("sex Female", line)) else "male"
previous_infection_stage = sft.get_char("from ", line)
if sum(count_chronic_daily) == 0:
success = False
report_file.write("Found no Chronic case in data.\n")
if sum(count_recovered_daily) == 0:
success = False
report_file.write("Found no Recovered case in data.\n")
if (not re.search("from subclinical", line)) and (not re.search("from acute", line)) and \
(not re.search("from Subclinical", line)) and (not re.search("from Acute", line)) and \
(not re.search("from SubClinical", line)):
success = False
if re.search("just went chronic", line):
report_file.write("BAD: individual age {0}, sex {1} went to Chronic state from {2} state, "
"expected Acute state or SubClinical state."
"\n".format(age, sex, previous_infection_stage))
else:
ind_id = int(sft.get_val("Individual ", line))
report_file.write("BAD: individual {0} age {1}, sex {2} went to Susceptible state from {3}"
" state, expected Acute state or SubClinical state."
"\n".format(ind_id, age, sex, previous_infection_stage))
if success:
report_file.write(sft.format_success_msg(success))
# not much to graph, we're checking if individuals who go chronic or recover do so from the correct state
# no "theoretical" data as such. Currently plotting # of people who recovered and who went chronic daily
sft.plot_data_sorted(count_chronic_daily, count_recovered_daily, label1="Count Chronic Daily",
label2="Count Recovered Daily",
title="Daily counts of Chronic and Recovered",
xlabel="Time (days)", ylabel="Number of occurrences", category='acu_subc_to_chr_sus_transition')
