def test_immunity_within_age_range(ind_df,
max_age,
expected_immunity,
report_file_name=sft.sft_output_filename,
min_age=0):
'''
NOTE: This only works for binary immunity with a constant (same
at min as max) immunity probability in age range
:param ind_df:
:param max_age:
:param report_file: s_f_t.txt file
:param expected_immunity:
:param min_age:
:return:
'''
print("Min Age is: {0}".format(min_age))
print("Max Age is: {0}".format(max_age))
agerange_df = ind_df[(min_age < ind_df[DataframeKeys.AGE])
& (max_age > ind_df[DataframeKeys.AGE])]
immune = ind_df[DataframeKeys.MOD_ACQUIRE] == 0
immune_count = len(agerange_df[immune])
total_count = len(agerange_df)
category_name = "Expected immunity in ages {0} to {1}".format(
min_age, max_age)
with open(report_file_name, 'a') as report_file:
sft.test_binomial_95ci(immune_count, total_count, expected_immunity,
report_file, category_name)
def create_report_file(param_obj, stdout_data_obj, report_name, debug):
with open(report_name, "w") as outfile:
config_name = param_obj[KEY_CONFIG_NAME]
outfile.write("Config_name = {}\n".format(config_name))
success = True
fast_progressor_fraction = param_obj[KEY_FAST_PROGRESSOR_FRACTION]
results = []
for key, value in stdout_data_obj[0].items():
if key > 2 and value[
1] > 35: # ignoring the smaller latent #s because there's not enough data for the test
result = sft.test_binomial_95ci(
value[0], value[1], fast_progressor_fraction, outfile,
"Time {} :{} new exogenous infections for {} latent infections."
"\n".format(key, value[0], value[1]))
results.append(result)
if not results:
outfile.write("No results.\n")
success = False
elif results.count(
False
) > 2: # not enough data for a binomial test on the binomial results,
success = False # so, the closest we can come is "less than two False is good".
outfile.write(sft.format_success_msg(success))
sft.plot_data(stdout_data_obj[1],
title="Exogenous infections for tracked time steps",
category="exogenous_infections_tb_only")
if debug:
print("SUMMARY: Success={0}\n".format(success))
return success
def create_report_file(param_obj, output_dict, report_name, debug):
with open(report_name, "w") as outfile:
config_name = param_obj[KEY_CONFIG_NAME]
outfile.write("Config_name = {}\n".format(config_name))
success = True
slow_progressor_rate = param_obj[KEY_SLOW_PROGRESSOR_RATE]
fast_progressor_rate = param_obj[KEY_FAST_PROGRESSOR_RATE]
child_fast_fraction = param_obj[KEY_CHILD_FRACTION]
adult_fast_fraction = param_obj[KEY_ADULT_FRACTION]
progression_multiplier = np.mean(param_obj[KEY_CD4_PROGRESSION_MULTIPLIER])
simulation_duration = param_obj[KEY_DURATION]
if not len(output_dict):
success = False
outfile.write(sft.sft_no_test_data)
outfile.write("checking test conditions: \n")
if child_fast_fraction:
success = False
outfile.write("BAD: expected {0} = 0, got {1} from config.json. "
"Please fix the test.\n".format(KEY_CHILD_FRACTION, child_fast_fraction))
dist_exponential_np_slow = np.random.exponential(1/slow_progressor_rate, 100)
if min(dist_exponential_np_slow) < simulation_duration:
success = False
outfile.write("BAD: expected a small {0} to distinguish fast and slow progress TB, got {1} from config.json. Please "
"fix the test.\n".format(KEY_SLOW_PROGRESSOR_RATE, slow_progressor_rate))
outfile.write("conditional check result is {}.\n".format(success))
actual_timer = []
internal_timer = []
slow_count = 0
outfile.write("collecting the actual timestep between latent and presymptomatic:\n")
outfile.write("checking if the internal timer matches the PreSymptomatic to Cleared duration:\n")
for id in output_dict:
presymptomatic_time = timer = latent_time = None
if KEY_PRESYMPTOMATIC in output_dict[id]:
presymptomatic_time = output_dict[id][KEY_PRESYMPTOMATIC][0]
timer = output_dict[id][KEY_PRESYMPTOMATIC][1]
internal_timer.append(timer)
if KEY_LATENT in output_dict[id]:
latent_time = output_dict[id][KEY_LATENT]
if latent_time:
if presymptomatic_time: # some individual may not move to presymptomatic state at the end of the simulation
actual_timer.append(presymptomatic_time - latent_time)
if presymptomatic_time - latent_time != math.ceil(timer):
success = False
outfile.write("BAD: individual {0} has internal timer = {1} but the actual timer is {2} (enter "
"latent state at timestep {3}, enter presymptomatic active state at "
"timestep {4}).\n".format(id, timer, presymptomatic_time - latent_time,
latent_time, presymptomatic_time))
else:
slow_count += 1
if debug:
outfile.write("Individual {0} moved to latent state at timestep {1} and is not move to "
"presymptomatic active yet at the end of simulation (duration = {2})."
"\n".format(id, latent_time, simulation_duration))
else:
success = False
outfile.write("BAD: individual {0} moved to presymptomatic active state at timerstep {1} before entering "
"latent state.\n".format(id, presymptomatic_time))
if not len(actual_timer):
success = False
outfile.write("BAD: There is no latent to presymptomatic state transition in this test, please fix the test.\n")
outfile.write("Running ks test for latent to presymptomatic internal timer and numpy exponential distribution: \n")
size = len(internal_timer)
scale = 1.0 / fast_progressor_rate
dist_exponential_np = np.random.exponential(scale, size)
sft.plot_data_sorted(internal_timer, dist2=np.array(dist_exponential_np), label1="internal timer",
label2="numpy exponential", title="exponential rate = {}".format(fast_progressor_rate),
xlabel="data point", ylabel="latent to presymptomatic internal timer",
category='latent_to_presymptomatic_internal_timer', show=True, line=False, overlap=True)
result = sft.test_exponential(internal_timer, p1=fast_progressor_rate, report_file=outfile, integers=True, roundup=True,
round_nearest=False)
outfile.write("ks test result is {0}, exponential rate = {1}, # of data point = {2}.\n".format(result, fast_progressor_rate, size))
if not result:
success = False
outfile.write("BAD: test exponential for latent to presymptomatic duration failed with fast_progressor_rate "
"= {}.\n".format(fast_progressor_rate))
else:
outfile.write(
"GOOD: test exponential for latent to presymptomatic duration passed with fast_progressor_rate "
"= {}.\n".format(fast_progressor_rate))
outfile.write("running binomial test with 95% confidence for Fast_Progressor_Fraction_Adult:\n")
result2 = sft.test_binomial_95ci(num_success=len(internal_timer), num_trials=len(internal_timer) + slow_count,
prob=adult_fast_fraction * progression_multiplier, report_file=outfile,
category="Fast_Progressor_Fraction_Adult")
outfile.write("number of slow progressor is {0} and number of fast progressor is {1}.\n".format(slow_count, len(internal_timer)))
if not result2:
success = False
outfile.write("BAD: binomial test for Fast_Progressor_Fraction_Adult = {0} and TB_CD4_Primary_Progression= {1} failed"
".\n".format(adult_fast_fraction, progression_multiplier))
else:
outfile.write("GOOD: binomial test for Fast_Progressor_Fraction_Adult = {0} and TB_CD4_Primary_Progression= {1} passed"
".\n".format(adult_fast_fraction, progression_multiplier))
outfile.write(sft.format_success_msg(success))
if debug:
print( "SUMMARY: Success={0}\n".format(success) )
return success
def application(report_file):
# pdb.set_trace()
# print( "Post-processing: " + report_file )
sft.wait_for_done()
cdj = json.loads(open("config.json").read())["parameters"]
start_time = cdj["Start_Time"]
timestep = start_time
lines_c = []
count_contact = 0
with open("test.txt") as logfile:
for line in logfile:
if "Update(): Time:" in line:
# calculate time step
timestep = int(float(sft.get_val('Time: ', line)))
elif ("Exposing" in line) and ("route 'contact'" in line):
# collect dose_response probabilities and dose for route contact
line = "TimeStep: " + str(timestep) + " " + line
lines_c.append(line)
# route=0, outbreak; route=1, contact; route=2, environment
elif ("AcquireNewInfection:" in line) and ("route=1" in line):
count_contact += 1
success = True
infection_prob_c_all = []
infection_prob_c_theoretic_all = []
with open(sft.sft_output_filename, "w") as report_file:
if not lines_c:
success = False
report_file.write("Found no individual exposed from route contact.\n")
else:
prob_per_num_exposures = {}
prob_per_num_exposures_expected = {}
random_dose_response = None
for line in lines_c:
dose_response = float(sft.get_val("infects=", line))
# some version of logging as a typo
ind_id = int(sft.get_val("individual ", line)) if "individual " in line else \
int(sft.get_val("inividual ", line))
immunity = float(sft.get_val("immunity=", line))
num_exposures = float(sft.get_val("num_exposures=", line))
infection_prob = float(sft.get_val("prob=", line))
timestep = int(sft.get_val("TimeStep: ", line))
infection_prob_theoretic = 1.0 - math.pow((1.0 - immunity * dose_response), num_exposures)
infection_prob_c_all.append(infection_prob)
infection_prob_c_theoretic_all.append(round(infection_prob_theoretic, 6))
if math.fabs(infection_prob_theoretic - infection_prob) > 5e-2:
success = False
report_file.write("BAD: Infection probability for individual {0} at time {1}, route contact is {2},"
" expected {3}.\n".format(ind_id, timestep, infection_prob,
infection_prob_theoretic))
# pick a random dose_response, for plot to be added to spec
if random_dose_response is None:
random_dose_response = dose_response
if immunity == 1.0 and dose_response == random_dose_response:
if num_exposures not in prob_per_num_exposures:
prob_per_num_exposures[num_exposures] = infection_prob
prob_per_num_exposures_expected[num_exposures] = infection_prob_theoretic
if success:
report_file.write("GOOD: Infection probability matches expected value for every exposure.\n")
else:
report_file.write("BAD: Infection probability doesn't match expected value for every exposure.\n")
# plot for Spec
# sort dictionary by keys
prob_per_num_exposures = dict(sorted(prob_per_num_exposures.items()))
prob_per_num_exposures_expected = dict(sorted(prob_per_num_exposures_expected.items()))
sft.plot_data([v for k,v in prob_per_num_exposures.items()],
[v for k,v in prob_per_num_exposures_expected.items()],
label1="Actual",
label2="Expected", title="Infection Probability(Contact)\nimmunity=1, dose_response={}".format(random_dose_response),
xlabel="num_exposures",
ylabel="Infection Probability",
category='infection_probability_contact',
alpha=0.5, overlap=True, xticks=range(len(prob_per_num_exposures)),
xtickslabel=[str(int(i)) for i in prob_per_num_exposures]
)
# sum of independent Bernoulli trials is Poisson binomial distribution
# contact is calculated first
# mean = average number of events per interval
mean_c = sft.calc_poisson_binomial(infection_prob_c_all)['mean']
sd_c = sft.calc_poisson_binomial(infection_prob_c_all)['standard_deviation']
num_trials_c = len(infection_prob_c_all)
prob_c = mean_c / float(num_trials_c)
# report_file.write("contact:trials = {2}, num_infection = {3},prob_c = {4}, mean= {0}, sd = {1}.
# \n".format(mean_c, sd_c,num_trials_c,count_contact,prob_c))
# TODO: comment out line 101 to 115 once #2895 is fixed. we are going to test New Infections By Route
# channels in NewInfection SFT, we only test the logging in this test.
isj = json.loads(open("output/InsetChart.json").read())["Channels"]
new_infection_contact = isj["New Infections By Route (CONTACT)"]["Data"]
insetchart_total_contact_infection = sum(new_infection_contact)
message_template = "{0}: {1} is {2}, while total number of exposure = {3}, sum of contact " \
"infection_probability = {4}, expected total contact infections = {5} " \
"with standard_deviation = {6}.\n"
if not sft.test_binomial_95ci(insetchart_total_contact_infection, num_trials_c, prob_c, report_file, 'contact infection'):
success = False
report_file.write(message_template.format("BAD", "sum of 'New Infections By Route (CONTACT)' in "
"InsetChart.json", insetchart_total_contact_infection,
num_trials_c, prob_c, mean_c, sd_c))
else:
report_file.write(message_template.format("GOOD", "sum of 'New Infections By Route (CONTACT)' in "
"InsetChart.json", insetchart_total_contact_infection,
num_trials_c, prob_c, mean_c, sd_c))
if not sft.test_binomial_95ci(count_contact, num_trials_c, prob_c, report_file, 'contact infection'):
success = False
report_file.write(message_template.format("BAD", "total contact infections from StdOut logging",
count_contact,
num_trials_c, prob_c, mean_c, sd_c))
else:
report_file.write(message_template.format("GOOD", "total contact infections from StdOut logging",
count_contact,
num_trials_c, prob_c, mean_c, sd_c))
sft.plot_data(infection_prob_c_all, infection_prob_c_theoretic_all,
# filter(lambda a: a != 0, infection_prob_c_all),
# filter(lambda a: a != 0, infection_prob_c_theoretic_all),
label1="Actual",
label2="Expected", title="Infection Probability Contact",
xlabel="Occurrence",
ylabel="Infection Probability",
category='immunity_probability_contact',
alpha=0.5, overlap=True, sort=False)
report_file.write(sft.format_success_msg(success))
def create_report_file(self, param_obj, output_df, report_name=sft.sft_output_filename, age_index_1=0,
age_index_2=9):
"""
Test the probability to become Chronic state and recovered from Acute or Subclinical state, write the
test result to file.
:param param_obj: config parameter dictionary
:param output_df: dataframe which contains test data from stdout file
:param report_name: file to write test result
:param age_index_1: index to define age range to test
:param age_index_2: index to define age range to test
:return: True or False for test result
"""
# 4*10 list to store the count for cases [0][]: Chr_male, [1][]: Chr_female, [2][]: Sus_male. [3][]: Sus_female
count = [[0] * 9 for _ in range(4)]
tcp = param_obj[ConfigParameters.KEY_TCP]
gpag_male = Constant.gpag_male
gpag_female = Constant.gpag_female
success = True
with open(report_name, "w") as report_file:
# if len(output_df) == output_df.ix[len(output_df)-1, ConfigParameters.KEY_SIMULATION_TIMESTEP]:
if len(output_df) == 0:
success = False
report_file.write(sft.sft_no_test_data)
else:
time_steps = output_df[ConfigParameters.KEY_SIMULATION_TIMESTEP]
lines = output_df["Content"]
for index in range(len(output_df)):
time_step = time_steps[index]
line = lines[index]
age = float(sft.get_val(" age ", line))
sex = "female" if ("sex 1" in line or "sex Female" in line) else "male"
if "just went chronic" in line:
# to Chronic
# python 2.7 the (int / int) operator is integer division
i = int(age / 10)
# for age > 80, put them into the last age group
if i > 8:
i = 8
if sex == "male":
count[0][i] += 1
else:
count[1][i] += 1
else:
# to Susceptible
# python 2.7 the (int / int) operator is integer division
i = int(age / 10 )
# for age > 80, put them into the last age group
if i > 8:
i = 8
if sex == "male":
count[2][i] += 1
else:
count[3][i] += 1
# calculate theoretic probability of becoming a Chronic carrier in two 1*9 lists
theoretic_p_male = [x * tcp for x in gpag_male]
theoretic_p_female = [x * tcp for x in gpag_female]
# calculate actual probability of becoming a Chronic carrier in two 1*9 lists
actual_p_male = [x / float(x + y) if (x + y) != 0 else -1 for x, y in zip(count[0], count[2])]
actual_p_female = [x / float(x + y) if (x + y) != 0 else -1 for x, y in zip(count[1], count[3])]
for x in range(age_index_1, age_index_2):
age = Constant.age
# calculate the total chronic cases and sample sizes for Male and Female
actual_chr_count_male = count[0][x]
actual_count_male = count[0][x] + count[2][x]
actual_chr_count_female = count[1][x]
actual_count_female = count[1][x] + count[3][x]
# Male
category = 'sex: Male, age: ' + age[x]
if actual_count_male == 0:
success = False
report_file.write(
"Found no male in age group {0} went to Chronic state or was recovered from {1} state.\n".format(
age[x], self.from_state))
elif theoretic_p_male[x] < 5e-2 or theoretic_p_male[x] > 0.95:
# for cases that binomial confidence interval will not work: prob close to 0 or 1
if math.fabs(actual_p_male[x] - theoretic_p_male[x]) > 5e-2:
success = False
report_file.write(
"BAD: Proportion of {0} {1} cases that become Chronic is {2}, expected {3}.\n".format(
category, self.from_state, actual_p_male[x], theoretic_p_male[x]))
elif not sft.test_binomial_95ci(actual_chr_count_male, actual_count_male, theoretic_p_male[x],
report_file, category):
success = False
# Female
category = 'sex: Female, age: ' + age[x]
if actual_count_female == 0:
success = False
report_file.write(
"Found no female in age group {0} went to Chronic state or was recovered from {1} state.\n".format(
age[x], from_state))
elif theoretic_p_female[x] < 5e-2 or theoretic_p_female[x] > 0.95:
# for cases that binomial confidence interval will not work: prob close to 0 or 1
if math.fabs(actual_p_female[x] - theoretic_p_female[x]) > 5e-2:
success = False
report_file.write(
"BAD: Proportion of {0} {1} cases that become Chronic is {2}, expected {3}.\n".format(
category, from_state,actual_p_female[x], theoretic_p_female[x]))
elif not sft.test_binomial_95ci(actual_chr_count_female, actual_count_female, theoretic_p_female[x],
report_file, category):
success = False
report_file.write(sft.format_success_msg(success))
return success
def create_report_file(param_obj, campaign_obj, output_dict, report_dict,
report_name, debug):
with open(report_name, "w") as outfile:
config_name = param_obj[KEY_CONFIG_NAME]
outfile.write("Config_name = {}\n".format(config_name))
success = True
sensitivity = campaign_obj[KEY_BASE_SENSITIVITY]
treatment_fraction = campaign_obj[KEY_TREATMENT_FRACTION]
prob = sensitivity * treatment_fraction
binomial_test_count = 0
positive = []
total = []
if not len(report_dict):
success = False
outfile.write(sft.sft_no_test_data)
point_fail = 0
point_tolerance = 0.3
for t in report_dict:
num_success = report_dict[t][KEY_POSITIVE]
num_trials = output_dict[t][KEY_POSITIVE_HUMAN] + output_dict[t][
KEY_NEGATIVE_HUMAN]
positive.append(num_success)
total.append(num_trials)
# the logging includes positive, negative and default, please see issue #2279
if num_trials * prob < 5 or num_trials * (1 - prob) < 5:
outfile.write(
"At timestep {0}, there is not enough sample size : mean = {1}, sample size - mean = {2}"
".\n".format(t, num_trials * prob,
num_trials * (1 - prob)))
else:
result = sft.test_binomial_95ci(num_success,
num_trials,
prob,
report_file=outfile,
category="TB test positive")
outfile.write(
"At timestep {0}, the binomial 95% test result is {1}.\n".
format(t, result))
binomial_test_count += 1
if not result:
point_fail += 1
if not binomial_test_count:
success = False
outfile.write(
"BAD: There is not enough sample size for binomial test in every time step, please fix the test.\n"
)
point_test_message = f"Detected failures: {point_fail} trials: {binomial_test_count} tolerance: {point_tolerance}.\n"
if point_fail / binomial_test_count > point_tolerance:
success = False
outfile.write(f"FAIL: {point_test_message}")
else:
outfile.write(f"PASS: {point_test_message}")
outfile.write(
"BIG TEST: Testing the total of diagnoses across the simulation...\n"
)
total_result = sft.test_binomial_95ci(
num_success=sum(positive),
num_trials=sum(total),
prob=prob,
report_file=outfile,
category="Total Active TB test positive")
if not total_result:
success = False
outfile.write("FAIL: the total test failed, see line above\n")
else:
outfile.write("PASS: the total test ")
sft.plot_data(
positive,
dist2=total,
label1="TBTestPositive",
label2="Total tested",
title="Test positive vs. total, positive proportion = {}".format(
prob),
xlabel="time step",
ylabel="# of individuals",
category='Test_positive_vs_total',
show=True,
line=False)
# TODO: write test to check if report matches debug logging. Pending on #2279. May not need this part.
outfile.write(sft.format_success_msg(success))
if debug:
print("SUMMARY: Success={0}\n".format(success))
return success
def application(report_file):
#pdb.set_trace()
#print( "Post-processing: " + report_file )
sft.wait_for_done()
cdj = json.loads(open("config.json").read())["parameters"]
start_time = cdj["Start_Time"]
stat_pop = None
timestep = start_time
chronic_list = []
recovered_list = []
with open(sft.sft_test_filename) as logfile:
for line in logfile:
if re.search("Update\(\): Time:", line):
#calculate time step
timestep += 1
if stat_pop is None:
stat_pop = sft.get_val("StatPop: ", line)
if re.search("just went chronic", line) and re.search(
"from acute", line):
#append time step and all Infection stage transition to list
age = float(sft.get_val(" age ", line))
sex = "female" if ("sex 1" in line) or ("sex Female"
in line) else "male"
chronic_list.append((age, sex))
if re.search("just recovered", line) and re.search(
"from acute", line):
# append time step and all Infection stage transition to list
age = float(sft.get_val(" age ", line))
sex = "female" if ("sex 1" in line) or ("sex Female"
in line) else "male"
recovered_list.append((age, sex))
# 4*10 list to store the count for cases [0][]: Chr_male, [1][]: Chr_female, [2][]: Sus_male. [3][]: Sus_female
count = [[0] * 9 for _ in range(4)]
tcpm = cdj["Typhoid_Carrier_Probability_Male"]
tcpf = cdj["Typhoid_Carrier_Probability_Female"]
gpag_male = [0.0, 0.0, 0.045, 0.134, 0.167, 0.198, 0.247, 0.435, 0.4]
gpag_female = [0.0, 0.097, 0.234, 0.431, 0.517, 0.60, 0.692, 0.692, 0.555]
success = True
with open(sft.sft_output_filename, "w") as report_file:
if not len(chronic_list) or not len(recovered_list):
success = False
report_file.write(sft.sft_no_test_data)
else:
for age, sex in chronic_list:
# to Chronic
# python 2.7 the (int / int) operator is integer division
i = int(age) // 10
# for age > 80, put them into the last age group
if i > 8: i = 8
if sex == "male":
count[0][i] += 1
else:
count[1][i] += 1
for age, sex in recovered_list:
# to Susceptible
# python 2.7 the (int / int) operator is integer division
i = int(age) // 10
# for age > 80, put them into the last age group
if i > 8: i = 8
if sex == "male":
count[2][i] += 1
else:
count[3][i] += 1
# calculate theoretic probability of becoming a Chronic carrier in two 1*9 lists
theoretic_p_male = [x * tcpm for x in gpag_male]
theoretic_p_female = [x * tcpf for x in gpag_female]
# calculate actual probability of becoming a Chronic carrier in two 1*9 lists
actual_p_male = [
x / float(x + y) if (x + y) != 0 else -1
for x, y in zip(count[0], count[2])
]
actual_p_female = [
x / float(x + y) if (x + y) != 0 else -1
for x, y in zip(count[1], count[3])
]
age = [
"0-9", "10-19", "20-29", "30-39", "40-49", "50-59", "60-69",
"70-79", "80+"
]
for x in range(len(age)):
# calculate the total chronic cases and sample sizes for Male and Female
actual_chr_count_male = count[0][x]
actual_count_male = count[0][x] + count[2][x]
actual_chr_count_female = count[1][x]
actual_count_female = count[1][x] + count[3][x]
# Male
category = 'sex: Male, age: ' + age[x]
if actual_count_male == 0:
success = False
report_file.write(
"Found no male in age group {0} went to Chronic state or was recovered from Acute state.\n"
.format(age[x]))
elif theoretic_p_male[x] < 5e-2 or theoretic_p_male[x] > 0.95:
# for cases that binomial confidence interval will not work: prob close to 0 or 1
if math.fabs(actual_p_male[x] -
theoretic_p_male[x]) > 5e-2:
success = False
report_file.write(
"BAD: Proportion of {0} Acute cases that become Chronic is {1}, expected {2}.\n"
.format(category, actual_p_male[x],
theoretic_p_male[x]))
elif not sft.test_binomial_95ci(
actual_chr_count_male, actual_count_male,
theoretic_p_male[x], report_file, category):
success = False
#Female
category = 'sex: Female, age: ' + age[x]
if actual_count_female == 0:
success = False
report_file.write(
"Found no female in age group {0} went to Chronic state or was recovered from Acute state.\n"
.format(age[x]))
elif theoretic_p_female[x] < 5e-2 or theoretic_p_female[
x] > 0.95:
# for cases that binomial confidence interval will not work: prob close to 0 or 1
if math.fabs(actual_p_female[x] -
theoretic_p_female[x]) > 5e-2:
success = False
report_file.write(
"BAD: Proportion of {0} Acute cases that become Chronic is {1}, expected {2}.\n"
.format(category, actual_p_female[x],
theoretic_p_female[x]))
elif not sft.test_binomial_95ci(
actual_chr_count_female, actual_count_female,
theoretic_p_female[x], report_file, category):
success = False
xticks = np.arange(len(age))
sft.plot_data(actual_p_male,
theoretic_p_male,
label1="Actual Probability",
label2="Expected Probability",
title="Acute to Chronic Probability Male\n"
"Total population = {0}, TCPM = {1}".format(
stat_pop, tcpm),
xlabel="Age bins",
ylabel="Acute -> Chronic Probability",
category='acute_to_chronic_male',
overlap=True,
alpha=0.5,
xticks=xticks,
xtickslabel=age)
sft.plot_data(actual_p_female,
theoretic_p_female,
label1="Actual Probability",
label2="Expected Probability",
title="Acute to Chronic Probability Female\n"
"Total population = {0}, TCPF = {1}".format(
stat_pop, tcpf),
xlabel="Age bins",
ylabel="Acute -> Chronic Probability",
category='acute_to_chronic_female',
overlap=True,
alpha=0.5,
xticks=xticks,
xtickslabel=age)
if success:
report_file.write(sft.format_success_msg(success))
def application(report_file):
sft.wait_for_done()
# print( "Post-processing: " + report_file )
cdj = json.loads(open("config.json").read())["parameters"]
tsf = cdj["Typhoid_Symptomatic_Fraction"]
start_time = cdj["Start_Time"]
lines = []
timestep = start_time
with open("test.txt") as logfile:
for line in logfile:
if "Update(): Time:" in line:
# alculate time step
timestep += 1
elif "Infection stage transition" in line:
# append time step and all Infection stage transition to list
line = "TimeStep: " + str(timestep) + " " + line
lines.append(line)
success = True
with open(sft.sft_output_filename, "w") as report_file:
if not lines:
success = False
report_file.write("BAD: Found no data matching test case.\n")
else:
subcount = 0
acutecount = 0
for line in lines:
if ((not re.search("->SubClinical:", line) and not re.search("->Acute:", line)) and
re.search(", Prepatent->", line)) or \
(re.search("->SubClinical:", line) or re.search("->Acute:", line)) \
and not re.search(", Prepatent->", line):
success = False
ind_id = int(sft.get_val("Individual=", line))
current_infection_stage = get_char_after("->", line)
previous_infection_stage = get_char_before("->", line)
report_file.write(
"BAD: individuals {0} went to {1} state from {2} state, expected Prepatent->Acute "
"or Prepatent->SubClinical.\n".format(
ind_id, current_infection_stage,
previous_infection_stage))
elif ", Prepatent->Acute:" in line:
# count # of cases: from Prepatent to Acute
acutecount += 1
elif ", Prepatent->SubClinical:" in line:
# count # of cases: from Prepatent to Subclinical
subcount += 1
actual_tsf = acutecount / float(subcount + acutecount)
if subcount + acutecount == 0:
success = False
report_file.write(
"Bad:Found no individual exits Prepatent state in log.\n")
else:
# for cases that binomial confidence interval will not work
if tsf < 1e-2 or tsf > 0.99:
actual_tsf = acutecount / float(subcount + acutecount)
if math.fabs(actual_tsf - tsf) > 5e-2:
success = False
report_file.write(
"BAD: Proportion of prepatent cases that become acute vs. subclinical is {0} "
"instead of {1}. Actual Acute case = {2} vs. Actual SubClinical case = {3}."
"\n".format(actual_tsf, tsf, acutecount, subcount))
# use binomial 95% confidence interval
elif not sft.test_binomial_95ci(
acutecount, subcount + acutecount, tsf, report_file,
'PrePatent to Acute transition'):
success = False
report_file.write("acute_count = {0} and subc_count = {1}.\n".format(
acutecount, subcount))
report_file.write(sft.format_success_msg(success))
# for spec
fig = plt.figure()
plt.bar(np.arange(2),
height=[acutecount, subcount],
width=0.5,
align='center',
alpha=0.5)
plt.title("Typhoid Symptomatic Fraction = {}".format(tsf))
plt.xticks(np.arange(2),
['Prepatent->Acute', 'Prepatent->SubClinical'])
for a, b in zip(np.arange(2), [acutecount, subcount]):
plt.text(a,
b,
str(b),
horizontalalignment='center',
verticalalignment='top'
) #, bbox=dict(facecolor='red', alpha=0.1))
plt.ylabel("Occurrence")
if sft.check_for_plotting():
plt.show()
fig.savefig("typhoid_symptomatic_fraction.png")
