def test_immunity_within_age_range(ind_df, max_age, expected_immunity, report_file_name=sft.sft_output_filename, min_age=0): ''' NOTE: This only works for binary immunity with a constant (same at min as max) immunity probability in age range :param ind_df: :param max_age: :param report_file: s_f_t.txt file :param expected_immunity: :param min_age: :return: ''' print("Min Age is: {0}".format(min_age)) print("Max Age is: {0}".format(max_age)) agerange_df = ind_df[(min_age < ind_df[DataframeKeys.AGE]) & (max_age > ind_df[DataframeKeys.AGE])] immune = ind_df[DataframeKeys.MOD_ACQUIRE] == 0 immune_count = len(agerange_df[immune]) total_count = len(agerange_df) category_name = "Expected immunity in ages {0} to {1}".format( min_age, max_age) with open(report_file_name, 'a') as report_file: sft.test_binomial_95ci(immune_count, total_count, expected_immunity, report_file, category_name)
def create_report_file(param_obj, stdout_data_obj, report_name, debug): with open(report_name, "w") as outfile: config_name = param_obj[KEY_CONFIG_NAME] outfile.write("Config_name = {}\n".format(config_name)) success = True fast_progressor_fraction = param_obj[KEY_FAST_PROGRESSOR_FRACTION] results = [] for key, value in stdout_data_obj[0].items(): if key > 2 and value[ 1] > 35: # ignoring the smaller latent #s because there's not enough data for the test result = sft.test_binomial_95ci( value[0], value[1], fast_progressor_fraction, outfile, "Time {} :{} new exogenous infections for {} latent infections." "\n".format(key, value[0], value[1])) results.append(result) if not results: outfile.write("No results.\n") success = False elif results.count( False ) > 2: # not enough data for a binomial test on the binomial results, success = False # so, the closest we can come is "less than two False is good". outfile.write(sft.format_success_msg(success)) sft.plot_data(stdout_data_obj[1], title="Exogenous infections for tracked time steps", category="exogenous_infections_tb_only") if debug: print("SUMMARY: Success={0}\n".format(success)) return success
def create_report_file(param_obj, output_dict, report_name, debug): with open(report_name, "w") as outfile: config_name = param_obj[KEY_CONFIG_NAME] outfile.write("Config_name = {}\n".format(config_name)) success = True slow_progressor_rate = param_obj[KEY_SLOW_PROGRESSOR_RATE] fast_progressor_rate = param_obj[KEY_FAST_PROGRESSOR_RATE] child_fast_fraction = param_obj[KEY_CHILD_FRACTION] adult_fast_fraction = param_obj[KEY_ADULT_FRACTION] progression_multiplier = np.mean(param_obj[KEY_CD4_PROGRESSION_MULTIPLIER]) simulation_duration = param_obj[KEY_DURATION] if not len(output_dict): success = False outfile.write(sft.sft_no_test_data) outfile.write("checking test conditions: \n") if child_fast_fraction: success = False outfile.write("BAD: expected {0} = 0, got {1} from config.json. " "Please fix the test.\n".format(KEY_CHILD_FRACTION, child_fast_fraction)) dist_exponential_np_slow = np.random.exponential(1/slow_progressor_rate, 100) if min(dist_exponential_np_slow) < simulation_duration: success = False outfile.write("BAD: expected a small {0} to distinguish fast and slow progress TB, got {1} from config.json. Please " "fix the test.\n".format(KEY_SLOW_PROGRESSOR_RATE, slow_progressor_rate)) outfile.write("conditional check result is {}.\n".format(success)) actual_timer = [] internal_timer = [] slow_count = 0 outfile.write("collecting the actual timestep between latent and presymptomatic:\n") outfile.write("checking if the internal timer matches the PreSymptomatic to Cleared duration:\n") for id in output_dict: presymptomatic_time = timer = latent_time = None if KEY_PRESYMPTOMATIC in output_dict[id]: presymptomatic_time = output_dict[id][KEY_PRESYMPTOMATIC][0] timer = output_dict[id][KEY_PRESYMPTOMATIC][1] internal_timer.append(timer) if KEY_LATENT in output_dict[id]: latent_time = output_dict[id][KEY_LATENT] if latent_time: if presymptomatic_time: # some individual may not move to presymptomatic state at the end of the simulation actual_timer.append(presymptomatic_time - latent_time) if presymptomatic_time - latent_time != math.ceil(timer): success = False outfile.write("BAD: individual {0} has internal timer = {1} but the actual timer is {2} (enter " "latent state at timestep {3}, enter presymptomatic active state at " "timestep {4}).\n".format(id, timer, presymptomatic_time - latent_time, latent_time, presymptomatic_time)) else: slow_count += 1 if debug: outfile.write("Individual {0} moved to latent state at timestep {1} and is not move to " "presymptomatic active yet at the end of simulation (duration = {2})." "\n".format(id, latent_time, simulation_duration)) else: success = False outfile.write("BAD: individual {0} moved to presymptomatic active state at timerstep {1} before entering " "latent state.\n".format(id, presymptomatic_time)) if not len(actual_timer): success = False outfile.write("BAD: There is no latent to presymptomatic state transition in this test, please fix the test.\n") outfile.write("Running ks test for latent to presymptomatic internal timer and numpy exponential distribution: \n") size = len(internal_timer) scale = 1.0 / fast_progressor_rate dist_exponential_np = np.random.exponential(scale, size) sft.plot_data_sorted(internal_timer, dist2=np.array(dist_exponential_np), label1="internal timer", label2="numpy exponential", title="exponential rate = {}".format(fast_progressor_rate), xlabel="data point", ylabel="latent to presymptomatic internal timer", category='latent_to_presymptomatic_internal_timer', show=True, line=False, overlap=True) result = sft.test_exponential(internal_timer, p1=fast_progressor_rate, report_file=outfile, integers=True, roundup=True, round_nearest=False) outfile.write("ks test result is {0}, exponential rate = {1}, # of data point = {2}.\n".format(result, fast_progressor_rate, size)) if not result: success = False outfile.write("BAD: test exponential for latent to presymptomatic duration failed with fast_progressor_rate " "= {}.\n".format(fast_progressor_rate)) else: outfile.write( "GOOD: test exponential for latent to presymptomatic duration passed with fast_progressor_rate " "= {}.\n".format(fast_progressor_rate)) outfile.write("running binomial test with 95% confidence for Fast_Progressor_Fraction_Adult:\n") result2 = sft.test_binomial_95ci(num_success=len(internal_timer), num_trials=len(internal_timer) + slow_count, prob=adult_fast_fraction * progression_multiplier, report_file=outfile, category="Fast_Progressor_Fraction_Adult") outfile.write("number of slow progressor is {0} and number of fast progressor is {1}.\n".format(slow_count, len(internal_timer))) if not result2: success = False outfile.write("BAD: binomial test for Fast_Progressor_Fraction_Adult = {0} and TB_CD4_Primary_Progression= {1} failed" ".\n".format(adult_fast_fraction, progression_multiplier)) else: outfile.write("GOOD: binomial test for Fast_Progressor_Fraction_Adult = {0} and TB_CD4_Primary_Progression= {1} passed" ".\n".format(adult_fast_fraction, progression_multiplier)) outfile.write(sft.format_success_msg(success)) if debug: print( "SUMMARY: Success={0}\n".format(success) ) return success
def application(report_file): # pdb.set_trace() # print( "Post-processing: " + report_file ) sft.wait_for_done() cdj = json.loads(open("config.json").read())["parameters"] start_time = cdj["Start_Time"] timestep = start_time lines_c = [] count_contact = 0 with open("test.txt") as logfile: for line in logfile: if "Update(): Time:" in line: # calculate time step timestep = int(float(sft.get_val('Time: ', line))) elif ("Exposing" in line) and ("route 'contact'" in line): # collect dose_response probabilities and dose for route contact line = "TimeStep: " + str(timestep) + " " + line lines_c.append(line) # route=0, outbreak; route=1, contact; route=2, environment elif ("AcquireNewInfection:" in line) and ("route=1" in line): count_contact += 1 success = True infection_prob_c_all = [] infection_prob_c_theoretic_all = [] with open(sft.sft_output_filename, "w") as report_file: if not lines_c: success = False report_file.write("Found no individual exposed from route contact.\n") else: prob_per_num_exposures = {} prob_per_num_exposures_expected = {} random_dose_response = None for line in lines_c: dose_response = float(sft.get_val("infects=", line)) # some version of logging as a typo ind_id = int(sft.get_val("individual ", line)) if "individual " in line else \ int(sft.get_val("inividual ", line)) immunity = float(sft.get_val("immunity=", line)) num_exposures = float(sft.get_val("num_exposures=", line)) infection_prob = float(sft.get_val("prob=", line)) timestep = int(sft.get_val("TimeStep: ", line)) infection_prob_theoretic = 1.0 - math.pow((1.0 - immunity * dose_response), num_exposures) infection_prob_c_all.append(infection_prob) infection_prob_c_theoretic_all.append(round(infection_prob_theoretic, 6)) if math.fabs(infection_prob_theoretic - infection_prob) > 5e-2: success = False report_file.write("BAD: Infection probability for individual {0} at time {1}, route contact is {2}," " expected {3}.\n".format(ind_id, timestep, infection_prob, infection_prob_theoretic)) # pick a random dose_response, for plot to be added to spec if random_dose_response is None: random_dose_response = dose_response if immunity == 1.0 and dose_response == random_dose_response: if num_exposures not in prob_per_num_exposures: prob_per_num_exposures[num_exposures] = infection_prob prob_per_num_exposures_expected[num_exposures] = infection_prob_theoretic if success: report_file.write("GOOD: Infection probability matches expected value for every exposure.\n") else: report_file.write("BAD: Infection probability doesn't match expected value for every exposure.\n") # plot for Spec # sort dictionary by keys prob_per_num_exposures = dict(sorted(prob_per_num_exposures.items())) prob_per_num_exposures_expected = dict(sorted(prob_per_num_exposures_expected.items())) sft.plot_data([v for k,v in prob_per_num_exposures.items()], [v for k,v in prob_per_num_exposures_expected.items()], label1="Actual", label2="Expected", title="Infection Probability(Contact)\nimmunity=1, dose_response={}".format(random_dose_response), xlabel="num_exposures", ylabel="Infection Probability", category='infection_probability_contact', alpha=0.5, overlap=True, xticks=range(len(prob_per_num_exposures)), xtickslabel=[str(int(i)) for i in prob_per_num_exposures] ) # sum of independent Bernoulli trials is Poisson binomial distribution # contact is calculated first # mean = average number of events per interval mean_c = sft.calc_poisson_binomial(infection_prob_c_all)['mean'] sd_c = sft.calc_poisson_binomial(infection_prob_c_all)['standard_deviation'] num_trials_c = len(infection_prob_c_all) prob_c = mean_c / float(num_trials_c) # report_file.write("contact:trials = {2}, num_infection = {3},prob_c = {4}, mean= {0}, sd = {1}. # \n".format(mean_c, sd_c,num_trials_c,count_contact,prob_c)) # TODO: comment out line 101 to 115 once #2895 is fixed. we are going to test New Infections By Route # channels in NewInfection SFT, we only test the logging in this test. isj = json.loads(open("output/InsetChart.json").read())["Channels"] new_infection_contact = isj["New Infections By Route (CONTACT)"]["Data"] insetchart_total_contact_infection = sum(new_infection_contact) message_template = "{0}: {1} is {2}, while total number of exposure = {3}, sum of contact " \ "infection_probability = {4}, expected total contact infections = {5} " \ "with standard_deviation = {6}.\n" if not sft.test_binomial_95ci(insetchart_total_contact_infection, num_trials_c, prob_c, report_file, 'contact infection'): success = False report_file.write(message_template.format("BAD", "sum of 'New Infections By Route (CONTACT)' in " "InsetChart.json", insetchart_total_contact_infection, num_trials_c, prob_c, mean_c, sd_c)) else: report_file.write(message_template.format("GOOD", "sum of 'New Infections By Route (CONTACT)' in " "InsetChart.json", insetchart_total_contact_infection, num_trials_c, prob_c, mean_c, sd_c)) if not sft.test_binomial_95ci(count_contact, num_trials_c, prob_c, report_file, 'contact infection'): success = False report_file.write(message_template.format("BAD", "total contact infections from StdOut logging", count_contact, num_trials_c, prob_c, mean_c, sd_c)) else: report_file.write(message_template.format("GOOD", "total contact infections from StdOut logging", count_contact, num_trials_c, prob_c, mean_c, sd_c)) sft.plot_data(infection_prob_c_all, infection_prob_c_theoretic_all, # filter(lambda a: a != 0, infection_prob_c_all), # filter(lambda a: a != 0, infection_prob_c_theoretic_all), label1="Actual", label2="Expected", title="Infection Probability Contact", xlabel="Occurrence", ylabel="Infection Probability", category='immunity_probability_contact', alpha=0.5, overlap=True, sort=False) report_file.write(sft.format_success_msg(success))
def create_report_file(self, param_obj, output_df, report_name=sft.sft_output_filename, age_index_1=0, age_index_2=9): """ Test the probability to become Chronic state and recovered from Acute or Subclinical state, write the test result to file. :param param_obj: config parameter dictionary :param output_df: dataframe which contains test data from stdout file :param report_name: file to write test result :param age_index_1: index to define age range to test :param age_index_2: index to define age range to test :return: True or False for test result """ # 4*10 list to store the count for cases [0][]: Chr_male, [1][]: Chr_female, [2][]: Sus_male. [3][]: Sus_female count = [[0] * 9 for _ in range(4)] tcp = param_obj[ConfigParameters.KEY_TCP] gpag_male = Constant.gpag_male gpag_female = Constant.gpag_female success = True with open(report_name, "w") as report_file: # if len(output_df) == output_df.ix[len(output_df)-1, ConfigParameters.KEY_SIMULATION_TIMESTEP]: if len(output_df) == 0: success = False report_file.write(sft.sft_no_test_data) else: time_steps = output_df[ConfigParameters.KEY_SIMULATION_TIMESTEP] lines = output_df["Content"] for index in range(len(output_df)): time_step = time_steps[index] line = lines[index] age = float(sft.get_val(" age ", line)) sex = "female" if ("sex 1" in line or "sex Female" in line) else "male" if "just went chronic" in line: # to Chronic # python 2.7 the (int / int) operator is integer division i = int(age / 10) # for age > 80, put them into the last age group if i > 8: i = 8 if sex == "male": count[0][i] += 1 else: count[1][i] += 1 else: # to Susceptible # python 2.7 the (int / int) operator is integer division i = int(age / 10 ) # for age > 80, put them into the last age group if i > 8: i = 8 if sex == "male": count[2][i] += 1 else: count[3][i] += 1 # calculate theoretic probability of becoming a Chronic carrier in two 1*9 lists theoretic_p_male = [x * tcp for x in gpag_male] theoretic_p_female = [x * tcp for x in gpag_female] # calculate actual probability of becoming a Chronic carrier in two 1*9 lists actual_p_male = [x / float(x + y) if (x + y) != 0 else -1 for x, y in zip(count[0], count[2])] actual_p_female = [x / float(x + y) if (x + y) != 0 else -1 for x, y in zip(count[1], count[3])] for x in range(age_index_1, age_index_2): age = Constant.age # calculate the total chronic cases and sample sizes for Male and Female actual_chr_count_male = count[0][x] actual_count_male = count[0][x] + count[2][x] actual_chr_count_female = count[1][x] actual_count_female = count[1][x] + count[3][x] # Male category = 'sex: Male, age: ' + age[x] if actual_count_male == 0: success = False report_file.write( "Found no male in age group {0} went to Chronic state or was recovered from {1} state.\n".format( age[x], self.from_state)) elif theoretic_p_male[x] < 5e-2 or theoretic_p_male[x] > 0.95: # for cases that binomial confidence interval will not work: prob close to 0 or 1 if math.fabs(actual_p_male[x] - theoretic_p_male[x]) > 5e-2: success = False report_file.write( "BAD: Proportion of {0} {1} cases that become Chronic is {2}, expected {3}.\n".format( category, self.from_state, actual_p_male[x], theoretic_p_male[x])) elif not sft.test_binomial_95ci(actual_chr_count_male, actual_count_male, theoretic_p_male[x], report_file, category): success = False # Female category = 'sex: Female, age: ' + age[x] if actual_count_female == 0: success = False report_file.write( "Found no female in age group {0} went to Chronic state or was recovered from {1} state.\n".format( age[x], from_state)) elif theoretic_p_female[x] < 5e-2 or theoretic_p_female[x] > 0.95: # for cases that binomial confidence interval will not work: prob close to 0 or 1 if math.fabs(actual_p_female[x] - theoretic_p_female[x]) > 5e-2: success = False report_file.write( "BAD: Proportion of {0} {1} cases that become Chronic is {2}, expected {3}.\n".format( category, from_state,actual_p_female[x], theoretic_p_female[x])) elif not sft.test_binomial_95ci(actual_chr_count_female, actual_count_female, theoretic_p_female[x], report_file, category): success = False report_file.write(sft.format_success_msg(success)) return success
def create_report_file(param_obj, campaign_obj, output_dict, report_dict, report_name, debug): with open(report_name, "w") as outfile: config_name = param_obj[KEY_CONFIG_NAME] outfile.write("Config_name = {}\n".format(config_name)) success = True sensitivity = campaign_obj[KEY_BASE_SENSITIVITY] treatment_fraction = campaign_obj[KEY_TREATMENT_FRACTION] prob = sensitivity * treatment_fraction binomial_test_count = 0 positive = [] total = [] if not len(report_dict): success = False outfile.write(sft.sft_no_test_data) point_fail = 0 point_tolerance = 0.3 for t in report_dict: num_success = report_dict[t][KEY_POSITIVE] num_trials = output_dict[t][KEY_POSITIVE_HUMAN] + output_dict[t][ KEY_NEGATIVE_HUMAN] positive.append(num_success) total.append(num_trials) # the logging includes positive, negative and default, please see issue #2279 if num_trials * prob < 5 or num_trials * (1 - prob) < 5: outfile.write( "At timestep {0}, there is not enough sample size : mean = {1}, sample size - mean = {2}" ".\n".format(t, num_trials * prob, num_trials * (1 - prob))) else: result = sft.test_binomial_95ci(num_success, num_trials, prob, report_file=outfile, category="TB test positive") outfile.write( "At timestep {0}, the binomial 95% test result is {1}.\n". format(t, result)) binomial_test_count += 1 if not result: point_fail += 1 if not binomial_test_count: success = False outfile.write( "BAD: There is not enough sample size for binomial test in every time step, please fix the test.\n" ) point_test_message = f"Detected failures: {point_fail} trials: {binomial_test_count} tolerance: {point_tolerance}.\n" if point_fail / binomial_test_count > point_tolerance: success = False outfile.write(f"FAIL: {point_test_message}") else: outfile.write(f"PASS: {point_test_message}") outfile.write( "BIG TEST: Testing the total of diagnoses across the simulation...\n" ) total_result = sft.test_binomial_95ci( num_success=sum(positive), num_trials=sum(total), prob=prob, report_file=outfile, category="Total Active TB test positive") if not total_result: success = False outfile.write("FAIL: the total test failed, see line above\n") else: outfile.write("PASS: the total test ") sft.plot_data( positive, dist2=total, label1="TBTestPositive", label2="Total tested", title="Test positive vs. total, positive proportion = {}".format( prob), xlabel="time step", ylabel="# of individuals", category='Test_positive_vs_total', show=True, line=False) # TODO: write test to check if report matches debug logging. Pending on #2279. May not need this part. outfile.write(sft.format_success_msg(success)) if debug: print("SUMMARY: Success={0}\n".format(success)) return success
def application(report_file): #pdb.set_trace() #print( "Post-processing: " + report_file ) sft.wait_for_done() cdj = json.loads(open("config.json").read())["parameters"] start_time = cdj["Start_Time"] stat_pop = None timestep = start_time chronic_list = [] recovered_list = [] with open(sft.sft_test_filename) as logfile: for line in logfile: if re.search("Update\(\): Time:", line): #calculate time step timestep += 1 if stat_pop is None: stat_pop = sft.get_val("StatPop: ", line) if re.search("just went chronic", line) and re.search( "from acute", line): #append time step and all Infection stage transition to list age = float(sft.get_val(" age ", line)) sex = "female" if ("sex 1" in line) or ("sex Female" in line) else "male" chronic_list.append((age, sex)) if re.search("just recovered", line) and re.search( "from acute", line): # append time step and all Infection stage transition to list age = float(sft.get_val(" age ", line)) sex = "female" if ("sex 1" in line) or ("sex Female" in line) else "male" recovered_list.append((age, sex)) # 4*10 list to store the count for cases [0][]: Chr_male, [1][]: Chr_female, [2][]: Sus_male. [3][]: Sus_female count = [[0] * 9 for _ in range(4)] tcpm = cdj["Typhoid_Carrier_Probability_Male"] tcpf = cdj["Typhoid_Carrier_Probability_Female"] gpag_male = [0.0, 0.0, 0.045, 0.134, 0.167, 0.198, 0.247, 0.435, 0.4] gpag_female = [0.0, 0.097, 0.234, 0.431, 0.517, 0.60, 0.692, 0.692, 0.555] success = True with open(sft.sft_output_filename, "w") as report_file: if not len(chronic_list) or not len(recovered_list): success = False report_file.write(sft.sft_no_test_data) else: for age, sex in chronic_list: # to Chronic # python 2.7 the (int / int) operator is integer division i = int(age) // 10 # for age > 80, put them into the last age group if i > 8: i = 8 if sex == "male": count[0][i] += 1 else: count[1][i] += 1 for age, sex in recovered_list: # to Susceptible # python 2.7 the (int / int) operator is integer division i = int(age) // 10 # for age > 80, put them into the last age group if i > 8: i = 8 if sex == "male": count[2][i] += 1 else: count[3][i] += 1 # calculate theoretic probability of becoming a Chronic carrier in two 1*9 lists theoretic_p_male = [x * tcpm for x in gpag_male] theoretic_p_female = [x * tcpf for x in gpag_female] # calculate actual probability of becoming a Chronic carrier in two 1*9 lists actual_p_male = [ x / float(x + y) if (x + y) != 0 else -1 for x, y in zip(count[0], count[2]) ] actual_p_female = [ x / float(x + y) if (x + y) != 0 else -1 for x, y in zip(count[1], count[3]) ] age = [ "0-9", "10-19", "20-29", "30-39", "40-49", "50-59", "60-69", "70-79", "80+" ] for x in range(len(age)): # calculate the total chronic cases and sample sizes for Male and Female actual_chr_count_male = count[0][x] actual_count_male = count[0][x] + count[2][x] actual_chr_count_female = count[1][x] actual_count_female = count[1][x] + count[3][x] # Male category = 'sex: Male, age: ' + age[x] if actual_count_male == 0: success = False report_file.write( "Found no male in age group {0} went to Chronic state or was recovered from Acute state.\n" .format(age[x])) elif theoretic_p_male[x] < 5e-2 or theoretic_p_male[x] > 0.95: # for cases that binomial confidence interval will not work: prob close to 0 or 1 if math.fabs(actual_p_male[x] - theoretic_p_male[x]) > 5e-2: success = False report_file.write( "BAD: Proportion of {0} Acute cases that become Chronic is {1}, expected {2}.\n" .format(category, actual_p_male[x], theoretic_p_male[x])) elif not sft.test_binomial_95ci( actual_chr_count_male, actual_count_male, theoretic_p_male[x], report_file, category): success = False #Female category = 'sex: Female, age: ' + age[x] if actual_count_female == 0: success = False report_file.write( "Found no female in age group {0} went to Chronic state or was recovered from Acute state.\n" .format(age[x])) elif theoretic_p_female[x] < 5e-2 or theoretic_p_female[ x] > 0.95: # for cases that binomial confidence interval will not work: prob close to 0 or 1 if math.fabs(actual_p_female[x] - theoretic_p_female[x]) > 5e-2: success = False report_file.write( "BAD: Proportion of {0} Acute cases that become Chronic is {1}, expected {2}.\n" .format(category, actual_p_female[x], theoretic_p_female[x])) elif not sft.test_binomial_95ci( actual_chr_count_female, actual_count_female, theoretic_p_female[x], report_file, category): success = False xticks = np.arange(len(age)) sft.plot_data(actual_p_male, theoretic_p_male, label1="Actual Probability", label2="Expected Probability", title="Acute to Chronic Probability Male\n" "Total population = {0}, TCPM = {1}".format( stat_pop, tcpm), xlabel="Age bins", ylabel="Acute -> Chronic Probability", category='acute_to_chronic_male', overlap=True, alpha=0.5, xticks=xticks, xtickslabel=age) sft.plot_data(actual_p_female, theoretic_p_female, label1="Actual Probability", label2="Expected Probability", title="Acute to Chronic Probability Female\n" "Total population = {0}, TCPF = {1}".format( stat_pop, tcpf), xlabel="Age bins", ylabel="Acute -> Chronic Probability", category='acute_to_chronic_female', overlap=True, alpha=0.5, xticks=xticks, xtickslabel=age) if success: report_file.write(sft.format_success_msg(success))
def application(report_file): sft.wait_for_done() # print( "Post-processing: " + report_file ) cdj = json.loads(open("config.json").read())["parameters"] tsf = cdj["Typhoid_Symptomatic_Fraction"] start_time = cdj["Start_Time"] lines = [] timestep = start_time with open("test.txt") as logfile: for line in logfile: if "Update(): Time:" in line: # alculate time step timestep += 1 elif "Infection stage transition" in line: # append time step and all Infection stage transition to list line = "TimeStep: " + str(timestep) + " " + line lines.append(line) success = True with open(sft.sft_output_filename, "w") as report_file: if not lines: success = False report_file.write("BAD: Found no data matching test case.\n") else: subcount = 0 acutecount = 0 for line in lines: if ((not re.search("->SubClinical:", line) and not re.search("->Acute:", line)) and re.search(", Prepatent->", line)) or \ (re.search("->SubClinical:", line) or re.search("->Acute:", line)) \ and not re.search(", Prepatent->", line): success = False ind_id = int(sft.get_val("Individual=", line)) current_infection_stage = get_char_after("->", line) previous_infection_stage = get_char_before("->", line) report_file.write( "BAD: individuals {0} went to {1} state from {2} state, expected Prepatent->Acute " "or Prepatent->SubClinical.\n".format( ind_id, current_infection_stage, previous_infection_stage)) elif ", Prepatent->Acute:" in line: # count # of cases: from Prepatent to Acute acutecount += 1 elif ", Prepatent->SubClinical:" in line: # count # of cases: from Prepatent to Subclinical subcount += 1 actual_tsf = acutecount / float(subcount + acutecount) if subcount + acutecount == 0: success = False report_file.write( "Bad:Found no individual exits Prepatent state in log.\n") else: # for cases that binomial confidence interval will not work if tsf < 1e-2 or tsf > 0.99: actual_tsf = acutecount / float(subcount + acutecount) if math.fabs(actual_tsf - tsf) > 5e-2: success = False report_file.write( "BAD: Proportion of prepatent cases that become acute vs. subclinical is {0} " "instead of {1}. Actual Acute case = {2} vs. Actual SubClinical case = {3}." "\n".format(actual_tsf, tsf, acutecount, subcount)) # use binomial 95% confidence interval elif not sft.test_binomial_95ci( acutecount, subcount + acutecount, tsf, report_file, 'PrePatent to Acute transition'): success = False report_file.write("acute_count = {0} and subc_count = {1}.\n".format( acutecount, subcount)) report_file.write(sft.format_success_msg(success)) # for spec fig = plt.figure() plt.bar(np.arange(2), height=[acutecount, subcount], width=0.5, align='center', alpha=0.5) plt.title("Typhoid Symptomatic Fraction = {}".format(tsf)) plt.xticks(np.arange(2), ['Prepatent->Acute', 'Prepatent->SubClinical']) for a, b in zip(np.arange(2), [acutecount, subcount]): plt.text(a, b, str(b), horizontalalignment='center', verticalalignment='top' ) #, bbox=dict(facecolor='red', alpha=0.1)) plt.ylabel("Occurrence") if sft.check_for_plotting(): plt.show() fig.savefig("typhoid_symptomatic_fraction.png")