def parse_output_file(output_filename="test.txt", debug=False): """ creates a dictionary to store filtered information for each time step :param output_filename: file to parse (test.txt) :return: output_dict """ filtered_lines = [] with open(output_filename) as logfile: for line in logfile: if sft.has_match(line, [matches[0]]): filtered_lines.append(line) if debug: with open("DEBUG_filtered_lines.txt", "w") as outfile: outfile.writelines(filtered_lines) # initialize variables time_step = 0 core = None output_dict = {} for line in filtered_lines: time_step = sft.get_val(matches[0], line) core = sft.get_val(matches[1], line) if time_step not in output_dict: output_dict[time_step] = [core] else: output_dict[time_step].append(core) if debug: res_path = r'./DEBUG_core_per_time_step.json' with open(res_path, "w") as file: json.dump(output_dict, file, indent=4) return output_dict
def parse_stdout_file(stdout_filename="StdOut.txt", simulation_timestep=1, debug=False): """ creates a dictionary to store filtered information for each time step :param output_filename: file to parse (StdOut.txt) :return: stdout_df """ filtered_lines = [] with open(stdout_filename) as logfile: for line in logfile: if sft.has_match(line,matches): filtered_lines.append(line) if debug: with open("DEBUG_filtered_lines.txt", "w") as outfile: outfile.writelines(filtered_lines) # initialize variables time_step = index = 0 stdout_df = pd.DataFrame(columns=[ConfigKeys.Simulation_Timestep, Stdout.stat_pop, Stdout.infected]) stdout_df.index.name = 'index' for line in filtered_lines: if matches[0] in line: stat_pop = int(sft.get_val(Stdout.stat_pop, line)) infected = int(sft.get_val(Stdout.infected, line)) stdout_df.loc[index] = [time_step, stat_pop, infected] index += 1 time_step += simulation_timestep if debug: res_path = r'./DEBUG_filtered_from_logging.csv' stdout_df.to_csv(res_path) return stdout_df
def create_report_file(data): report_name = data[0] lines = data[1] tb_cd4_activation_vector = data[ 2] # this test assumes the vector is constant latency_update_data = [] original_latency_data = [] success = True with open(report_name, "w") as outfile: if not lines: outfile.write("BAD: No relevant test data found.\n") success = False for line in lines: if "Incubation_timer calculated as" in line: incubation_timer = float(sft.get_val("as ", line)) original_latency_data.append(incubation_timer) if "LifeCourseLatencyTimerUpdate" in line: new_incubation_timer = float(sft.get_val("timer ", line)) latency_update_data.append(new_incubation_timer) # expecting the original distribution to NOT match the art-triggered update distribution if sft.test_exponential(original_latency_data, tb_cd4_activation_vector[2], integers=True, roundup=True, round_nearest=False): outfile.write( "BAD: The updated latency data matches the original distribution.\n" ) success = False expected_update_data = np.random.exponential( 1 / tb_cd4_activation_vector[2], len(latency_update_data)) if not sft.test_exponential(latency_update_data, tb_cd4_activation_vector[2], outfile, integers=True, roundup=True, round_nearest=False): # as it should fail , success = bad. outfile.write( "BAD: The updated latency data does not match the expected distribution.\n" ) success = False outfile.write("Data points checked = {}.\n".format( len(latency_update_data))) outfile.write("SUMMARY: Success={0}\n".format(success)) sft.plot_data(sorted(latency_update_data), sorted(expected_update_data), label1="Actual", label2="Expected", title="Latency Duration recalculated for ART", xlabel="Data Points", ylabel="Days", category="tb_activation_and_cd4_hiv_first_on_art", line=True, overlap=True)
def parse_stdout_file(initial_timestep=0, stdout_filename="test.txt", debug=False): """ :param initial_timestep: first timestep from config :param stdout_filename: file to parse (test.txt) :param debug: whether or not we write an additional file that's full of the matched lines :return: array of lines of interest """ exogenous = "EXOGENOUS infected" # "EXOGENOUS infected" infectiousness = "UpdateInfectiousness" state_latent = "state= Latent" filtered_lines = [] exogenous_infected_count = 0 latent_count = 0 exogenous_infected_dict = {} exogenous_infected_list = [] infections_start_time = 12 # this will vary depending when imported people will begin to be infectious. initial_population = 0 # placeholder for the first time step update_time = "Update(): Time: " time = initial_timestep with open(stdout_filename) as logfile: for line in logfile: if update_time in line: if time > infections_start_time: exogenous_infected_list.append(exogenous_infected_count) exogenous_infected_dict[time] = [ exogenous_infected_count, latent_count ] time += 1 if time == 1: initial_population = int(sft.get_val("StatPop: ", line)) exogenous_infected_count = 0 # resetting for the next time step latent_count = 0 filtered_lines.append(line) elif exogenous in line: ind_id = int(float(sft.get_val("Individual ", line))) if ind_id <= initial_population: # ignoring imported people exogenous_infected_count += 1 filtered_lines.append(line) elif infectiousness in line and state_latent in line: ind_id = int(float(sft.get_val("Individual ", line))) fast_progressor = int(float(sft.get_val("progressor=", line))) filtered_lines.append(line) if not fast_progressor and ind_id <= initial_population: # ignoring imported people latent_count += 1 if debug: with open("filtered_lines.txt", "w") as outfile: for line in filtered_lines: outfile.write(line) return [exogenous_infected_dict, exogenous_infected_list]
def parse_output_file(output_filename="test.txt", debug=False): """ creates a dictionary to store stat populations, infected population, and MDR test result for each time step :param output_filename: file to parse (test.txt) :return: output_dict """ filtered_lines = [] with open(output_filename) as logfile: for line in logfile: if sft.has_match(line, matches): filtered_lines.append(line) if debug: with open("DEBUG_filtered_lines.txt", "w") as outfile: outfile.writelines(filtered_lines) # initialize variables time_step = 0 infected = 0 statpop = 0 simulation_timestep = 1 positive = 0 negative = 0 default = 0 output_dict = {} for line in filtered_lines: if matches[0] in line: output_dict[time_step] = { KEY_STAT_POP: statpop, KEY_INFECTED: infected, KEY_POSITIVE: positive, KEY_NEGATIVE: negative, KEY_DEFAULT: default } infected = sft.get_val(matches[2], line) statpop = sft.get_val(matches[1], line) time_step += simulation_timestep positive = 0 negative = 0 default = 0 if matches[3] in line: result = int(sft.get_val(matches[3], line)) if result: positive += 1 else: negative += 1 if matches[4] in line: default += 1 res_path = r'./tb_test_result_from_logging.json' with open(res_path, "w") as file: json.dump(output_dict, file, indent=4) return output_dict
def parse_output_file(output_filename="test.txt", simulation_timestep=1, debug=False): """ creates a dataframe of time step, infected, infectiouness and stat populations :param output_filename: file to parse (test.txt) :param simulation_timestep: simulation time step, * days :return: output_df: data frame contains: 1, time step, 2, # of infected population, 3, infectiousness, 4, statistical populations, at each time step """ filtered_lines = [] with open(output_filename) as logfile: for line in logfile: if sft.has_match(line, matches): filtered_lines.append(line) if debug: with open("filtered_lines.txt", "w") as outfile: outfile.writelines(filtered_lines) # initialize variables time_step = 0 # infectiousness = 0 infected = 0 statpop = 0 output_df = pd.DataFrame(columns=[ KEY_SIMULATION_TIMESTEP, KEY_INFECTED, KEY_INFECTIOUSNESS, KEY_STAT_POP ]) output_df.index.name = "index" index = 0 for line in filtered_lines: if matches[0] in line: infected = sft.get_val(matches[1], line) statpop = sft.get_val(matches[3], line) output_df.loc[index] = [ time_step, infected, infectiousness, statpop ] index += 1 time_step += simulation_timestep infectiousness = 0 continue if matches[2] in line: infectiousness = sft.get_val(matches[2], line) continue res_path = r'./infected_vs_infectiousness.csv' if not os.path.exists(os.path.dirname(res_path)): os.makedirs(os.path.dirname(res_path)) output_df.to_csv(res_path) return output_df
def parse_stdout_file(stdout_filename="StdOut.txt", simulation_timestep=1, debug=False): """ creates a dictionary to store filtered information for each time step :param output_filename: file to parse (StdOut.txt) :return: stdout_df """ filtered_lines = [] with open(stdout_filename) as logfile: for line in logfile: if sft.has_match(line, matches): filtered_lines.append(line) if debug: with open("DEBUG_filtered_lines.txt", "w") as outfile: outfile.writelines(filtered_lines) # initialize variables time_step = index = 0 stdout_df = pd.DataFrame(columns=[ ConfigKeys.Simulation_Timestep, Stdout.stat_pop, Stdout.infected, Stdout.group_id, Stdout.contagion, Stdout.prob ]) stdout_df.index.name = 'index' stat_pop = infected = contagion = prob = group_id = None group_contagion = {} for line in filtered_lines: if matches[0] in line: stat_pop = int(sft.get_val(Stdout.stat_pop, line)) infected = int(sft.get_val(Stdout.infected, line)) for group_id in sorted(group_contagion): stdout_df.loc[index] = [ time_step, stat_pop, infected, group_id, group_contagion[group_id][0], group_contagion[group_id][1] ] index += 1 group_contagion = {} time_step += simulation_timestep elif len(group_contagion) < 10 and matches[1] in line: contagion = float(sft.get_val(matches[1], line)) prob = float(sft.get_val(Stdout.prob, line)) group_id = int(sft.get_val(matches[2], line)) if group_id not in group_contagion: group_contagion[group_id] = [contagion, prob] if debug: res_path = r'./DEBUG_filtered_from_logging.csv' stdout_df.to_csv(res_path) return stdout_df
def parse_output_file(output_filename="test.txt", debug=False): """ creates a dictionary to store filtered information for each time step :param output_filename: file to parse (test.txt) :return: output_dict """ filtered_lines = [] with open(output_filename) as logfile: for line in logfile: if sft.has_match(line, matches): filtered_lines.append(line) if debug: with open("DEBUG_filtered_lines.txt", "w") as outfile: outfile.writelines(filtered_lines) # initialize variables time_step = 0 simulation_timestep = 1 output_dict = {} for line in filtered_lines: if matches[0] in line: time_step += simulation_timestep elif matches[1] in line: # this individual is PreSymptomatic active individual_id = sft.get_val(KEY_INDIVIDUAL, line) timer = float(sft.get_val(KEY_TIMER, line)) if individual_id in output_dict: output_dict[individual_id][KEY_PRESYMPTOMATIC] = [ time_step, timer ] else: output_dict[individual_id] = { KEY_PRESYMPTOMATIC: [time_step, timer] } elif matches[2] in line: # this individual is Symptomatic active individual_id = sft.get_val(KEY_INDIVIDUAL, line) timer = float(sft.get_val(KEY_TIMER, line)) if individual_id in output_dict: output_dict[individual_id][KEY_SYMPTOMATIC] = [ time_step, timer ] else: output_dict[individual_id] = { KEY_SYMPTOMATIC: [time_step, timer] } res_path = r'./SEIR_Presymptomatic_from_logging.json' with open(res_path, "w") as file: json.dump(output_dict, file, indent=4) return output_dict
def parse_stdout_file(curr_timestep=0, stdout_filename="test.txt", debug=False): """ :param curr_timestep: first timestep from config :param stdout_filename: file to parse (test.txt) :param debug: whether or not we write an additional file that's full of the matched lines :return: array of lines of interest """ expose = "Expose: Individual" filtered_lines = [] update_time = "Update(): Time:" time = 0 with open(stdout_filename) as logfile: for line in logfile: if update_time in line: time += 1 if expose in line: ind = int(sft.get_val("Individual ", line)) if ind <= 100: # do not look at imported people new_line = sft.add_time_stamp(time, line) filtered_lines.append(new_line) if debug: with open("filtered_lines.txt", "w") as outfile: outfile.writelines(filtered_lines) return filtered_lines
def parse_stdout_file(self, initial_timestep=0, stdout_filename="test.txt", debug=False): """ :param initial_timestep: first timestep from config :param stdout_filename: file to parse (test.txt) :param debug: whether or not we write an additional file that's full of the matched lines :return: array of attenuation for each day """ attenuation = [] filtered_lines = [] time_step = initial_timestep latest_time = initial_timestep with open(stdout_filename) as logfile: for line in logfile: if "Time: " in line: time_step += 1 if self.attenuation_text in line: if time_step > latest_time: attenuation.append( float(sft.get_val(self.attenuation_text, line))) latest_time = time_step if debug: filtered_lines.append(line) if debug: with open("filtered_lines.txt", "w") as outfile: for line in filtered_lines: outfile.write(line) return attenuation
def calc_expected_infectiousness(line, treatment_list): """ calculate the infectiousness based on the state and the params in config.json. """ states = ["state PRE.", "state SUB.", "state CHR.", "state ACU."] expected_infectiousness = None state = None for regex in states: match = re.search(regex, line) if match: if regex == "state PRE.": state = "Prepatent" expected_infectiousness = tai * tpri elif regex == "state SUB.": state = "Subclinical" expected_infectiousness = tai * tsri elif regex == "state CHR.": state = "Chronic" expected_infectiousness = tai * tcri elif regex == "state ACU.": state = "Acute" expected_infectiousness = tai ind_id = int(sft.get_val("Individual ", line)) if ind_id in treatment_list: expected_infectiousness *= treatment_multiplier return { 'expected_infectiousness': expected_infectiousness, 'state': state }
def parse_stdout_file(time_step=0, stdout_filename="test.txt", filtered_filename="filtered_lines.txt", debug=False): """ creates a list of total susceptible people daily (each day is new susceptible people + total susceptible the day before) :param time_step: first timestep from config :param stdout_filename: file to parse (test.txt) :param filtered_filename: file to which we write lines of interest when debug = True :param debug: flad indicating whether or not we want debug data written out :return: a list of total people susceptible each day, total population """ with open(stdout_filename, "r") as infile: filtered_lines = [] total_susceptible_daily = [] population = 0 total_susceptible = 0 for line in infile: if UPDATE_TIME in line: filtered_lines.append(line) total_susceptible_daily.append(total_susceptible) time_step += 1 if population == 0: population = int(sft.get_val(POPULATION, line)) elif SUSCEPTIBLE in line: filtered_lines.append(line) total_susceptible += 1 if debug: with open(filtered_filename, "w") as outfile: outfile.writelines(filtered_lines) return [total_susceptible_daily, population]
def create_report_file(data, debug=False): report_name = data[0] lines = data[1] tb_cd4_activation_vector = data[2] # this test assumes the vector is constant latency_data = {} duration_data = {} success = True with open(report_name, "w") as outfile: if not lines: outfile.write("BAD: No relevant test data found.\n") success = False for line in lines: if "LifeCourseLatencyTimerUpdate" in line: ind_id = int(sft.get_val("Individual ", line)) start_time_stamp = int(sft.get_val("time= ", line)) if ind_id in latency_data.keys(): outfile.write("Individual {} incubation timer reset at time {}. Please check. " "\n".format(ind_id, start_time_stamp)) latency_data[ind_id] = start_time_stamp elif "TBActivationPresymptomatic" in line: ind_id = int(sft.get_val("Individual ", line)) end_time_stamp = int(sft.get_val("time= ", line)) if ind_id not in latency_data.keys(): outfile.write("Individual {} went presymptomatic without incubation timer update at time {}. " "Please check. \n".format(ind_id, end_time_stamp)) else: duration = end_time_stamp - latency_data.get(ind_id) duration_data[ind_id] = duration if debug: with open("DEBUG_duration_data.json","w") as debug_outfile: json.dump(duration_data, debug_outfile, indent=4) durations = list(duration_data.values()) if not sft.test_exponential(durations, tb_cd4_activation_vector[0], outfile, integers=True, roundup=True, round_nearest=False): success = False outfile.write("Data points checked = {}.\n".format(len(duration_data))) outfile.write("SUMMARY: Success={0}\n".format(success)) # only used for graphing purposes expected_data = map(math.ceil, np.random.exponential(1/tb_cd4_activation_vector[0], len(duration_data))) expected_durations = list(expected_data) sft.plot_data_sorted(durations, expected_durations, label1="Actual", label2="Expected", title="Recalculated Latency Duration TB then HIV(Sorted)", xlabel="Data Points", ylabel="Days", category="tb_activation_and_cd4_tb_first", line = True, overlap=True)
def create_report_file(data, inset_days, report_name=sft.sft_output_filename): lines_c = data[0] lines_e = data[1] typhoid_acute_infectiousness = data[2] ALPHA = 0.175 N50 = 1.11e6 dose_response_data_c = [] dose_response_data_e = [] dose_response_data_theoretic_c = [] dose_response_data_theoretic_e = [] success = True with open(report_name, "w") as report_file: if len(lines_c) == 0: success = False report_file.write("Found no individual exposed from route contact.\n" ) else: for line in lines_c: dose_response_c = float(sft.get_val("infects=", line)) dose_response_data_c.append(dose_response_c) ind_id_c = int(sft.get_val("individual ", line)) timestep_c = int(sft.get_val("TimeStep: ", line)) fContact = float(sft.get_val("fContact=", line)) dose_response_theoretic_c = float(fContact / typhoid_acute_infectiousness) dose_response_data_theoretic_c.append(dose_response_theoretic_c) if math.fabs(dose_response_theoretic_c - dose_response_c) > 5e-2: success = False report_file.write( "BAD: Dose-response probability for individual {0} at time {1}, route contact is {2}, " "expected {3}.\n".format(ind_id_c, timestep_c, dose_response_c, dose_response_theoretic_c)) if len(lines_e) == 0: success = False report_file.write("Found no individual exposed from route environment.\n") else: for line in lines_e: dose_response_e = float(sft.get_val("infects=", line)) dose_response_data_e.append(dose_response_e) ind_id = int(sft.get_val("individual ", line)) timestep = int(sft.get_val("TimeStep: ", line)) exposure = float(sft.get_val("exposure=", line)) dose_response_theoretic_e = 1.0 - math.pow(1.0 + exposure * (math.pow(2.0, 1.0 / ALPHA) - 1.0) / N50, -1.0 * ALPHA) dose_response_data_theoretic_e.append(dose_response_theoretic_e) if math.fabs(dose_response_theoretic_e - dose_response_e) > 5e-2: success = False report_file.write( "BAD: Dose-response probability for individual {0} at time {1}, route environment is {2}, " "expected {3}.\n".format(ind_id, timestep, dose_response_e, dose_response_theoretic_e)) report_file.write("Sample size of dose response is {0} for route contact and {1} for route environment." "\n".format(len(lines_c), len(lines_e))) report_file.write(sft.format_success_msg(success)) return [dose_response_data_c, dose_response_data_theoretic_c, dose_response_data_e, dose_response_data_theoretic_e]
def create_report_file(data): report_name = data[0] lines = data[1] coinfection_mortality_rate_off_art = data[2] coinfection_mortality_rate_on_art = data[3] died_of_coinfection = "died of CoInfection" state_active_symptomatic = "infectionstatechange TBActivation " time_to_death_data = [] active_infections_dictionary = {} success = True with open(report_name, "w") as outfile: if not lines: outfile.write("BAD: No relevant test data found.\n") success = False for line in lines: if died_of_coinfection in line: ind_id = int(sft.get_val("Individual ", line)) time_stamp = int(sft.get_val("time= ", line)) if ind_id in active_infections_dictionary.keys(): time_to_death_data.append(time_stamp - active_infections_dictionary[ind_id]) else: success = False outfile.write("BAD: Individual {} died of coinfection without going active, at time {}." "\n".format(ind_id, time_stamp)) elif state_active_symptomatic in line: ind_id = int(sft.get_val("Individual ", line)) start_time_stamp = int(sft.get_val("time= ", line)) if ind_id in active_infections_dictionary.keys(): outfile.write("Individual {} went active symptomatic while already being active symptomatic" "at time {}. \n".format(ind_id, start_time_stamp)) else: active_infections_dictionary[ind_id] = start_time_stamp # expected_data here only used for graphing purposes expected_data = map(int, np.random.exponential(1/coinfection_mortality_rate_on_art, len(time_to_death_data))) if not sft.test_exponential(time_to_death_data, coinfection_mortality_rate_on_art, outfile, integers=True, roundup=False, round_nearest=False): success = False outfile.write("Data points checked = {}.\n".format(len(time_to_death_data))) outfile.write("SUMMARY: Success={0}\n".format(success)) sft.plot_data(sorted(time_to_death_data), sorted(expected_data), label1="Actual", label2="Expected", title="Time from Smear Positive On ART TBHIV to Death", xlabel="Data Points", ylabel="Days", category="tbhiv_mortality_smear_positive_on_art",line = True, overlap=True)
def create_report_file(data): report_name = data[0] lines = data[1] cd4_strata = data[2] mod_array = data[3] actual_data = [] expected_data = [] success = True epsilon = 0.000002 with open(report_name, "w") as outfile: if not lines: outfile.write("BAD: No relevant test data found.\n") success = False for line in lines: cd4_count = float(sft.get_val("CD4count=", line)) cd4_mod_actual = float(sft.get_val("CD4mod=", line)) cd4_mod_expected = tb_cd4_susceptibility_calc( [mod_array, cd4_strata, cd4_count]) actual_data.append(cd4_mod_actual) expected_data.append(cd4_mod_expected) if abs(cd4_mod_actual - cd4_mod_expected) > epsilon: success = False outfile.write( "BAD: At Time: {} for Individual {} with CD4 count {} Expected susceptibility modifier " "was {}, but actual was {}.\n".format( sft.get_val("time= ", line), sft.get_val("Individual ", line), cd4_count, cd4_mod_expected, cd4_mod_actual)) outfile.write("Data points checked = {} .\n".format(len(lines))) outfile.write("SUMMARY: Success={0}\n".format(success)) sft.plot_data_sorted(actual_data, expected_data, label1="Actual", label2="Expected", title="Susceptibility Modifier", xlabel="Data Points", ylabel="Modifying Multiplier", category="tb_susceptibility_and_cd4", line=True, overlap=True)
def parse_stdout_file(stdout_filename="StdOut.txt", simulation_timestep=1, debug=False): """ creates a dictionary to store filtered information for each time step :param output_filename: file to parse (StdOut.txt) :return: stdout_df """ filtered_lines = [] with open(stdout_filename) as logfile: for line in logfile: if sft.has_match(line,matches): filtered_lines.append(line) if debug: with open("DEBUG_filtered_lines.txt", "w") as outfile: outfile.writelines(filtered_lines) # initialize variables time_step = index = 0 stdout_df = pd.DataFrame(columns=[ConfigKeys.Simulation_Timestep, Stdout.stat_pop, Stdout.infected, Stdout.group_id, Stdout.contagion, Stdout.prob]) stdout_df.index.name = 'index' stat_pop = infected = contagion = prob = group_id = None group_contagion = {} for line in filtered_lines: if matches[0] in line: stat_pop = int(sft.get_val(Stdout.stat_pop, line)) infected = int(sft.get_val(Stdout.infected, line)) for group_id in sorted(group_contagion): stdout_df.loc[index] = [time_step, stat_pop, infected, group_id, group_contagion[group_id][0], group_contagion[group_id][1]] index += 1 group_contagion = {} time_step += simulation_timestep elif len(group_contagion) < 10 and matches[1] in line: contagion = float(sft.get_val(matches[1], line)) prob = float(sft.get_val(Stdout.prob, line)) group_id = int(sft.get_val(matches[2], line)) if group_id not in group_contagion: group_contagion[group_id] = [contagion, prob] if debug: res_path = r'./DEBUG_filtered_from_logging.csv' stdout_df.to_csv(res_path) return stdout_df
def parse_stdout_file(output_filename="test.txt", debug=False): """ Reads stdout file and creates an individual ID indexed DataFrame of ages and mod_aquire :param output_filename: stdout filename (test.txt) :param debug: generate :return: dataframe of all individuals from a single timestep """ matches = [ "{} = ".format(DataframeKeys.MOD_ACQUIRE), "{} = ".format(DataframeKeys.AGE), "{} = ".format(DataframeKeys.ID), "Update(): Time: " ] filtered_lines = [] with open(output_filename) as logfile: for line in logfile: if sft.has_match(line, matches): if matches[-1] in line: break filtered_lines.append(line) if debug: with open("filtered_lines.txt", "w") as outfile: outfile.writelines(filtered_lines) individuals_df = pd.DataFrame( columns=[DataframeKeys.AGE, DataframeKeys.MOD_ACQUIRE]) individuals_df.index.name = 'index' for line in filtered_lines: age = float(sft.get_val(matches[1], line)) acquire = float(sft.get_val(matches[0], line)) id = int(sft.get_val(matches[2], line)) individuals_df.loc[id] = [age, acquire] if debug: with open("DEBUG_individuals_dataframe.csv", "w") as outfile: outfile.write(individuals_df.to_csv()) return individuals_df
def parse_stdout_file( start_timestep, duration_of_interest, stdout_filename="test.txt", debug=False): """creates cum_death and death_times array :param start_timestep: drug start time :param stdout_filename: file to parse (test.txt) :return: cum_deaths, death_times """ filtered_lines = [] with open(stdout_filename) as logfile: for line in logfile: if sft.has_match(line, matches): filtered_lines.append(line) if debug: with open("filtered_lines.txt", "w") as outfile: outfile.writelines(filtered_lines) # initialize variables cum_deaths = [] deaths = [] death_times = [] timestep = 0 cum = death_daily = infected = 0 infected_individuals = [] for line in filtered_lines: if "Time:" in line: timestep += 1 infected_individuals.append(infected) cum_deaths.append(cum) deaths.append(death_daily) death_daily = 0 infected = int(sft.get_val("Infected: ", line)) else: cum += 1 death_daily += 1 # cum_deaths[-1] = cum if timestep <= duration_of_interest + start_timestep: death_times.append( timestep-start_timestep ) if debug: with open("cum_deaths.txt","w") as file: file.writelines( str( cum_deaths ) ) with open("death_times.txt","w") as file: file.writelines( str( death_times ) ) return cum_deaths, deaths, infected_individuals, death_times
def parse_output_file(output_filename="test.txt", simulation_timestep=1, debug=False): """ creates a dictionary to store filtered information for each time step :param output_filename: file to parse (test.txt) :return: output_dict """ filtered_lines = [] with open(output_filename) as logfile: for line in logfile: if sft.has_match(line,matches): filtered_lines.append(line) if debug: with open("DEBUG_filtered_lines.txt", "w") as outfile: outfile.writelines(filtered_lines) # initialize variables time_step = 0 positive = 0 negative = 0 default = 0 output_df = pd.DataFrame(columns=[ReportColumn.negative, ReportColumn.default, ReportColumn.positive]) output_df.index.name = Config.simulation_timestep for line in filtered_lines: if matches[0] in line: output_df.loc[time_step] = pd.Series({ReportColumn.positive: positive, ReportColumn.negative: negative, ReportColumn.default: default}) time_step += simulation_timestep positive = 0 negative = 0 default = 0 if matches[1] in line: result = int(sft.get_val(matches[1], line)) if result: positive += 1 else: negative += 1 if matches[2] in line: default += 1 res_path = r'./DEBUG_tb_test_result_from_logging.csv' with open(res_path, "w") as file: output_df.to_csv(file) return output_df
def parse_output_file(output_filename="test.txt", debug=False): """ creates an object which contains the heterogeneity multiplier and infectiousness :param output_filename: file to parse (test.txt) :return: output_obj: heterogeneity multiplier and infectiousness for each infection """ filtered_lines = [] output_obj= {} for match in matches: output_obj[match] = [] with open(output_filename) as logfile: for line in logfile: for match in matches: if match in line: output_obj[match].append(float(sft.get_val(match, line))) filtered_lines.append(line) break if debug: with open("filtered_lines.txt", "w") as outfile: outfile.writelines(filtered_lines) return output_obj
def parse_output_file(output_filename="test.txt", debug=False): """ creates an object which contains the heterogeneity multiplier and infectiousness :param output_filename: file to parse (test.txt) :return: output_obj: heterogeneity multiplier and infectiousness for each infection """ filtered_lines = [] output_obj = {} for match in matches: output_obj[match] = [] with open(output_filename) as logfile: for line in logfile: for match in matches: if match in line: output_obj[match].append(float(sft.get_val(match, line))) filtered_lines.append(line) break if debug: with open("filtered_lines.txt", "w") as outfile: outfile.writelines(filtered_lines) return output_obj
def application(report_file): sft.wait_for_done() lines = [] with open("test.txt") as logfile: for line in logfile: if re.search("Initializing Typhoid immunity object", line) and re.search("age=0.000000", line): lines.append(line) success = True actual_newborn_immunity_data = [] expected_newborn_immunity_data = [] with open(sft.sft_output_filename, "w") as report_file: if not lines: success = False report_file.write("BAD: Found no data matching test case.\n") else: for line in lines: immunity = float(sft.get_val(" immunity modifier", line)) actual_newborn_immunity_data.append(immunity) expected_newborn_immunity_data.append(1) if immunity != 1.000: success = False report_file.write( "BAD: immunity for newborn={0} instead of 1.0\n". format(immunity)) report_file.write(sft.format_success_msg(success)) sft.plot_data_sorted(actual_newborn_immunity_data, expected_newborn_immunity_data, label1="Actual", label2="Expected", title="Newborn Immunity Data", xlabel="Individuals", ylabel="Immunity", category='multi_mode_intervention')
def application(report_file): # pdb.set_trace() # print( "Post-processing: " + report_file ) sft.wait_for_done() cdj = json.loads(open("config.json").read())["parameters"] start_time = cdj["Start_Time"] timestep = start_time lines_c = [] count_contact = 0 with open("test.txt") as logfile: for line in logfile: if "Update(): Time:" in line: # calculate time step timestep = int(float(sft.get_val('Time: ', line))) elif ("Exposing" in line) and ("route 'contact'" in line): # collect dose_response probabilities and dose for route contact line = "TimeStep: " + str(timestep) + " " + line lines_c.append(line) # route=0, outbreak; route=1, contact; route=2, environment elif ("AcquireNewInfection:" in line) and ("route=1" in line): count_contact += 1 success = True infection_prob_c_all = [] infection_prob_c_theoretic_all = [] with open(sft.sft_output_filename, "w") as report_file: if not lines_c: success = False report_file.write("Found no individual exposed from route contact.\n") else: prob_per_num_exposures = {} prob_per_num_exposures_expected = {} random_dose_response = None for line in lines_c: dose_response = float(sft.get_val("infects=", line)) # some version of logging as a typo ind_id = int(sft.get_val("individual ", line)) if "individual " in line else \ int(sft.get_val("inividual ", line)) immunity = float(sft.get_val("immunity=", line)) num_exposures = float(sft.get_val("num_exposures=", line)) infection_prob = float(sft.get_val("prob=", line)) timestep = int(sft.get_val("TimeStep: ", line)) infection_prob_theoretic = 1.0 - math.pow((1.0 - immunity * dose_response), num_exposures) infection_prob_c_all.append(infection_prob) infection_prob_c_theoretic_all.append(round(infection_prob_theoretic, 6)) if math.fabs(infection_prob_theoretic - infection_prob) > 5e-2: success = False report_file.write("BAD: Infection probability for individual {0} at time {1}, route contact is {2}," " expected {3}.\n".format(ind_id, timestep, infection_prob, infection_prob_theoretic)) # pick a random dose_response, for plot to be added to spec if random_dose_response is None: random_dose_response = dose_response if immunity == 1.0 and dose_response == random_dose_response: if num_exposures not in prob_per_num_exposures: prob_per_num_exposures[num_exposures] = infection_prob prob_per_num_exposures_expected[num_exposures] = infection_prob_theoretic if success: report_file.write("GOOD: Infection probability matches expected value for every exposure.\n") else: report_file.write("BAD: Infection probability doesn't match expected value for every exposure.\n") # plot for Spec # sort dictionary by keys prob_per_num_exposures = dict(sorted(prob_per_num_exposures.items())) prob_per_num_exposures_expected = dict(sorted(prob_per_num_exposures_expected.items())) sft.plot_data([v for k,v in prob_per_num_exposures.items()], [v for k,v in prob_per_num_exposures_expected.items()], label1="Actual", label2="Expected", title="Infection Probability(Contact)\nimmunity=1, dose_response={}".format(random_dose_response), xlabel="num_exposures", ylabel="Infection Probability", category='infection_probability_contact', alpha=0.5, overlap=True, xticks=range(len(prob_per_num_exposures)), xtickslabel=[str(int(i)) for i in prob_per_num_exposures] ) # sum of independent Bernoulli trials is Poisson binomial distribution # contact is calculated first # mean = average number of events per interval mean_c = sft.calc_poisson_binomial(infection_prob_c_all)['mean'] sd_c = sft.calc_poisson_binomial(infection_prob_c_all)['standard_deviation'] num_trials_c = len(infection_prob_c_all) prob_c = mean_c / float(num_trials_c) # report_file.write("contact:trials = {2}, num_infection = {3},prob_c = {4}, mean= {0}, sd = {1}. # \n".format(mean_c, sd_c,num_trials_c,count_contact,prob_c)) # TODO: comment out line 101 to 115 once #2895 is fixed. we are going to test New Infections By Route # channels in NewInfection SFT, we only test the logging in this test. isj = json.loads(open("output/InsetChart.json").read())["Channels"] new_infection_contact = isj["New Infections By Route (CONTACT)"]["Data"] insetchart_total_contact_infection = sum(new_infection_contact) message_template = "{0}: {1} is {2}, while total number of exposure = {3}, sum of contact " \ "infection_probability = {4}, expected total contact infections = {5} " \ "with standard_deviation = {6}.\n" if not sft.test_binomial_95ci(insetchart_total_contact_infection, num_trials_c, prob_c, report_file, 'contact infection'): success = False report_file.write(message_template.format("BAD", "sum of 'New Infections By Route (CONTACT)' in " "InsetChart.json", insetchart_total_contact_infection, num_trials_c, prob_c, mean_c, sd_c)) else: report_file.write(message_template.format("GOOD", "sum of 'New Infections By Route (CONTACT)' in " "InsetChart.json", insetchart_total_contact_infection, num_trials_c, prob_c, mean_c, sd_c)) if not sft.test_binomial_95ci(count_contact, num_trials_c, prob_c, report_file, 'contact infection'): success = False report_file.write(message_template.format("BAD", "total contact infections from StdOut logging", count_contact, num_trials_c, prob_c, mean_c, sd_c)) else: report_file.write(message_template.format("GOOD", "total contact infections from StdOut logging", count_contact, num_trials_c, prob_c, mean_c, sd_c)) sft.plot_data(infection_prob_c_all, infection_prob_c_theoretic_all, # filter(lambda a: a != 0, infection_prob_c_all), # filter(lambda a: a != 0, infection_prob_c_theoretic_all), label1="Actual", label2="Expected", title="Infection Probability Contact", xlabel="Occurrence", ylabel="Infection Probability", category='immunity_probability_contact', alpha=0.5, overlap=True, sort=False) report_file.write(sft.format_success_msg(success))
def application(report_file, debug=True): sft.wait_for_done() """ Parse this line from test.txt: 00:00:00 [0] [V] [IndividualTyphoid] amplification calculated as 0.997059: day of year=1, start=360.000000, end=365.000000, ramp_up=30.000000, ramp_down=170.000000, cutoff=160.000000. 00:00:00 [0] [V] [IndividualTyphoid] Exposing individual 2 age 8582.488281 on route 'environment': prob=0.000000, infects=0.000008, immunity=1.000000, num_exposures=0, exposure=0.997059, environment=1.000000, iv_mult=1.000000. """ #print( "Post-processing: " + report_file ) # get params from config.json cdj = json.loads( open( "config.json" ).read() )["parameters"] ncdr = cdj["Node_Contagion_Decay_Rate"] start_time=cdj["Start_Time"] lines = [] timestep=start_time count =0 amp = {} exposure = {} environment = {} cum_shedding = cum = 0 cum_shedding_all = {} Statpop = [] with open( "test.txt" ) as logfile: for line in logfile: if "Update(): Time:" in line: # store the accumulated shedding and reset the counter at the end of each time step. cum_shedding_all[timestep] = cum_shedding pop = float(sft.get_val("StatPop: ", line)) Statpop.append(pop) # environmental cp decay cum_shedding *= 1.0 - ncdr # resetting shedding variables shedding = 0 #calculate time step timestep += 1 line = "TimeStep: " + str(timestep) + " " + line lines.append(line) elif "amplification calculated as" in line: count += 1 line = "TimeStep: " + str(timestep) + " " + line lines.append(line) amp[timestep] = float(sft.get_val("amplification calculated as ", line)) elif ("Exposing" in line) and ("environment" in line): line = "TimeStep: " + str(timestep) + " " + line lines.append(line) if timestep not in exposure: exposure[timestep] = float(sft.get_val("exposure=", line)) environment[timestep] = float(sft.get_val("environment=", line)) elif ("depositing" in line) and ("route environment" in line): # get shedding of contact route and add it to accumulated shedding shedding = float(sft.get_val("depositing ", line)) cum_shedding += shedding line = "TimeStep: " + str(timestep) + " " + line lines.append(line) if debug: with open("DEBUG_filtered_line.txt","w") as filter_file: filter_file.write("".join(lines)) #more params from config file rud=cdj["Environmental_Ramp_Up_Duration"] rdd=cdj["Environmental_Ramp_Down_Duration"] ecd=cdj["Environmental_Cutoff_Days"] eps=cdj["Environmental_Peak_Start"] peak_duration=365-rud-rdd-ecd # for eps > 365 peak_starttime = eps % 365 peak_endtime = peak_starttime + peak_duration cutoff_starttime = peak_starttime + peak_duration + rdd #cutoff_endtime = peak_starttime + peak_duration + rdd + ecd success = True expected_e_contagion = {} environmental_amp = {} inset_chart_obj = general_support.parse_inset_chart("output", "InsetChart.json", insetkey_list=[InsetKeys.ChannelsKeys.Environmental_Contagion_Population]) with open( sft.sft_output_filename, "w" ) as report_file: report_file.write("Peak_Start={0}, Peak_Duration={1}, Peak_End={2}, Ramp_Down={3}, Ramp_Up={4}," "Cutoff_Start={5}, Cutoff_Duration={6}.\n".format(peak_starttime, peak_duration, peak_endtime, rdd, rud, cutoff_starttime, ecd)) if ncdr != 1: report_file.write("WARNING: Node_Contagion_Decay_Rate is {}, suggest to set it to 1.\n".format( ncdr )) if count == 0: success = False report_file.write( "Found no data matching test case.\n" ) elif peak_duration < 0: success = False report_file.write("BAD: Environmental peak duration should be larger or equal to 0, the actual value is {}." "\n The ramp durations and cutoff days need to follow a rule where: ramp_up_duration + " "ramp_down_duration + cutoff_days < 365.\n".format(peak_duration)) else: # adjust the times so that the ramp up starts at time 0, which means the cut off ends at time 0 too. adjust_time = peak_starttime - rud peak_starttime -= adjust_time peak_endtime -= adjust_time cutoff_starttime -= adjust_time # cutoff_endtime -= adjust_time with open("DEBUG_contagion_data.txt", "w") as debug_file: for t in range(timestep - 2): amplification = environmental_amplification = None if t in amp: TimeStep = t - adjust_time day_in_year = TimeStep%365 amplification = amp[t] # Environment Ramp Up if day_in_year < peak_starttime: environmental_amplification = day_in_year/float(rud) # Environment peak elif peak_starttime <= day_in_year <= peak_endtime: environmental_amplification = 1 # Environment Ramp Down elif peak_endtime < day_in_year < cutoff_starttime: environmental_amplification = (cutoff_starttime - day_in_year) / float(rdd) # Environment cutoff elif day_in_year >= cutoff_starttime: environmental_amplification = 0 if math.fabs(amplification - environmental_amplification) > 5e-2 * environmental_amplification: success =False report_file.write("BAD: at time {0}, day of year = {1}, the environmental amplification is {2}," " expected {3}.\n".format(t, t%365, amplification, environmental_amplification)) environmental_amp[t] = environmental_amplification if t in exposure: exposure_t = exposure[t] if amplification is not None: environment_contagion = cum_shedding_all[t-start_time] / Statpop[t -start_time] expected_exposure = environment_contagion * amplification if math.fabs(expected_exposure - exposure_t) > 5e-2 * expected_exposure: success =False report_file.write("BAD: at time {0}, day of year = {1}, the amount of environmental contagion " "that individual is exposed is {2}, expected {3}" ".\n".format(t, t%365, exposure_t, expected_exposure)) expected_e_contagion[t] = expected_exposure if debug: environment_t = environment[t] inserchart_contagion_e = inset_chart_obj \ [InsetKeys.ChannelsKeys.Environmental_Contagion_Population][t] debug_file.write("At time step {0}: environment={1}, exposure={2} from loggind, " "{3}={4} from InsetChart, expected value={5}." "accumulated shedding after decay={6} calculated from logging, " "environmental amplification={7}.\n".format( t, environment_t,exposure_t, InsetKeys.ChannelsKeys.Environmental_Contagion_Population, inserchart_contagion_e, expected_exposure, environment_contagion, amplification )) sft.plot_data(amp.values(), environmental_amp.values(), label1="Actual Seasonal Attenuation", label2="Expected Seasonal Attenuation", title="Seasonal Attenuation", xlabel="Time", ylabel="Attenuation", category='seasonal_attenuation') for t in range(len(amp)): if t not in exposure: exposure[t] = 0 expected_e_contagion[t] = 0 sft.plot_data([exposure[key] for key in sorted(exposure.keys())], [expected_e_contagion[key] for key in sorted(expected_e_contagion.keys())], label1="exposure contagion(from StdOut)", label2="Expected", title="Environmental Contagion", xlabel="Time", ylabel="Environmental Contagion", category='Environmental_Contagion') report_file.write( sft.format_success_msg( success ) )
def parse_stdout_file(drug_start_time, start_timestep, stdout_filename="test.txt", debug=False): """creates cum_inactivations, inactivation_times arrays :param drug_start_time: drug start time :param start_timestep: first timestep from config :param stdout_filename: file to parse (test.txt) :return: infected_mosquitoes_per_day dictionary, total_infections int """ filtered_lines = [] with open(stdout_filename) as logfile: for line in logfile: if sft.has_match(line, matches): filtered_lines.append(line) if debug: with open("filtered_lines.txt", "w") as outfile: outfile.writelines(filtered_lines) # initialize variables cum_inactivations = [] inactivation_times = [] timestep = 0 cum = inactivation_daily = infected = active = 0 infected_individuals = [] inactivations = [] active_count = [] reactivation_times = {} for line in filtered_lines: if matches[2] in line: cum_inactivations.append(cum) infected_individuals.append(infected) inactivations.append(inactivation_daily) active_count.append(active) infected = int(sft.get_val("Infected: ", line)) inactivation_daily = 0 timestep += 1 elif matches[0] in line: # deactivation: have to track individual individual = int(sft.get_val("TB drug deactivated my \(", line)) # if timestep <= duration_of_interest + start_timestep + drug_start_time: inactivation_time = timestep - start_timestep - drug_start_time if individual in reactivation_times: inactivation_time = timestep - reactivation_times[individual] reactivation_times.pop(individual) # not including the reactivations: there are some data point lost due to simulation end before timers end # inactivation_times.append( inactivation_time ) else: cum += 1 active -= 1 inactivation_daily += 1 inactivation_times.append(inactivation_time) elif matches[ 1] in line: # "progressing from Latent to Active Presymptomatic while on TB Drugs" # activation: have to track individual individual = int(sft.get_val("Individual ", line)) reactivation_times[individual] = timestep elif matches[3] in line: # move to active active += 1 else: #die from HIV individual = int(sft.get_val("individual ", line)) if individual in reactivation_times: active -= 1 reactivation_times.pop(individual) if debug: with open("Cum_Inactivations.txt", "w") as outfile: outfile.write(str(cum_inactivations)) with open("Inactivations.txt", "w") as outfile: outfile.write(str(inactivation_times)) print( "there are {} individual in reactivation state at the end of simulation.\n" .format(len(reactivation_times))) return cum_inactivations, inactivation_times, active_count, inactivations
def application(report_file, debug=False): sft.wait_for_done() """ Parse this line from test.txt: 00:00:00 [0] [V] [IndividualTyphoid] amplification calculated as 0.997059: day of year=1, start=360.000000, end=365.000000, ramp_up=30.000000, ramp_down=170.000000, cutoff=160.000000. 00:00:00 [0] [V] [IndividualTyphoid] Exposing individual 2 age 8582.488281 on route 'environment': prob=0.000000, infects=0.000008, immunity=1.000000, num_exposures=0, exposure=0.997059, environment=1.000000, iv_mult=1.000000. """ # print( "Post-processing: " + report_file ) # get params from config.json cdj = json.loads(open("config.json").read())["parameters"] start_time = cdj["Start_Time"] lines = [] timestep = start_time count = 0 with open("test.txt") as logfile: for line in logfile: if re.search("Update\(\): Time:", line): # calculate time step timestep += 1 line = "TimeStep: " + str(timestep) + " " + line lines.append(line) if re.search("amplification calculated as", line): count += 1 line = "TimeStep: " + str(timestep) + " " + line lines.append(line) if re.search("Exposing", line) and re.search("environment", line): line = "TimeStep: " + str(timestep) + " " + line lines.append(line) # more params from config file rud = cdj["Environmental_Ramp_Up_Duration"] rdd = cdj["Environmental_Ramp_Down_Duration"] ecd = cdj["Environmental_Cutoff_Days"] eps = cdj["Environmental_Peak_Start"] peak_duration = 365 - rud - rdd - ecd # for eps > 365 peak_starttime = eps % 365 peak_endtime = peak_starttime + peak_duration cutoff_starttime = peak_starttime + peak_duration + rdd success = True amp = [] environmental_amp = [] with open(sft.sft_output_filename, "w") as report_file: if count == 0: success = False report_file.write("Found no data matching test case.\n") elif peak_duration < 0: success = False report_file.write("BAD: Environmental peak duration should be larger or equal to 0, the actual value is {}." "\n The ramp durations and cutoff days need to follow a rule where: ramp_up_duration + " "ramp_down_duration + cutoff_days < 365.\n".format(peak_duration)) else: # adjust the times so that the ramp up starts at time 0, which means the cut off ends at time 0 too. adjust_time = peak_starttime - rud peak_starttime -= adjust_time peak_endtime -= adjust_time cutoff_starttime -= adjust_time for line in lines: if re.search("Update\(\): Time:", line): TimeStep = int(sft.get_val("TimeStep: ", line)) TimeStep -= adjust_time day_in_year = TimeStep % 365 elif re.search("amplification calculated as", line): amplification = float(sft.get_val("amplification calculated as ", line)) # Environment Ramp Up if day_in_year < peak_starttime: environmental_amplification = day_in_year/float(rud) # Environment peak elif peak_starttime <= day_in_year <= peak_endtime: environmental_amplification = 1 # Environment Ramp Down elif peak_endtime < day_in_year < cutoff_starttime: environmental_amplification = (cutoff_starttime - day_in_year) / float(rdd) # Environment cutoff elif day_in_year >= cutoff_starttime: environmental_amplification = 0 if math.fabs(amplification - environmental_amplification) > 5e-2: success = False TimeStep = int(sft.get_val("TimeStep: ", line)) report_file.write("BAD: at time {0}, day of year = {1}, the environmental amplification is {2}," " expected {3}.\n".format(TimeStep, day_in_year+adjust_time, amplification, environmental_amplification)) amp.append(amplification) environmental_amp.append(environmental_amplification) elif re.search("Exposing", line): ind_id = sft.get_val("individual ", line) environment = float(sft.get_val("environment=", line)) exposure = float(sft.get_val("exposure=", line)) expected_exposure = environment * amplification if math.fabs(expected_exposure - exposure) > 1e-2: success = False TimeStep = int(sft.get_val("TimeStep: ", line)) report_file.write("BAD: at time {0}, day of year = {1}, the amount of environmental contagion " "that individual (2) is exposed is {3}, expected {4}" ".\n".format(TimeStep, day_in_year + adjust_time, ind_id, exposure, expected_exposure)) sft.plot_data_sorted(amp, environmental_amp, label1="Actual Seasonal Attenuation", label2="Expected Seasonal Attenuation", title="Seasonal Attenuation Peak Equals RUD plus CD plus RDD", xlabel="Time", ylabel="Attenuation", category='seasonal_attenuation_peakstartequalsRUDplusCDplusRDD') report_file.write(sft.format_success_msg(success))
def create_report_file(data): report_name = data[0] lines = data[1] cd4_strata = data[2] mod_array = data[3] base_inf = data[4] presymp_mult = data[5] smear_neg_mult = data[6] latent_data_points = 0 smear_negative_data_points = 0 smear_positive_data_points = 0 presymptomatic_data_points = 0 extrapulmonary_data_points = 0 success = True epsilon = 0.000002 with open(report_name, "w") as outfile: if not lines: outfile.write("BAD: No relevant test data found.\n") success = False for line in lines: tot_inf_actual = float(sft.get_val("total_infectiousness= ", line)) cd4_count = float(sft.get_val("CD4count= ", line)) cd4_mod_actual = float(sft.get_val("CD4mod= ", line)) cd4_mod_expected = tb_cd4_infectiousness_calc([mod_array, cd4_strata, cd4_count]) if "Latent" in line: latent_data_points += 1 if tot_inf_actual != 0: success = False outfile.write("BAD, found Latent infection with total_infectiousness= {} at time= {}. " "Expected 0. \n".format(tot_inf_actual, sft.get_val("time= ", line))) elif "SmearNegative" in line: smear_negative_data_points += 1 tot_inf_expected = cd4_mod_expected * base_inf * smear_neg_mult if abs(tot_inf_expected - tot_inf_actual) > epsilon: success = False outfile.write("BAD, found SmearNegative infection with total_infectiousness= {} at time= {}, " "Expected {}. \n {} \n ".format(tot_inf_actual, sft.get_val("time= ", line), tot_inf_expected, line)) elif "SmearPositive" in line: smear_positive_data_points += 1 tot_inf_expected = cd4_mod_expected * base_inf if abs(tot_inf_expected - tot_inf_actual) > epsilon: success = False outfile.write("BAD, found SmearPositive infection with total_infectiousness= {} at time= {}, " "Expected {}. \n {} \n".format(tot_inf_actual, sft.get_val("time= ", line), tot_inf_expected, line)) elif "Presymptomatic" in line: presymptomatic_data_points += 1 tot_inf_expected = cd4_mod_expected * base_inf * presymp_mult if abs(tot_inf_expected - tot_inf_actual) > epsilon: success = False outfile.write("BAD, found Presymptomatic infection with total_infectiousness= {} at time= {}, " "Expected {}. \n {}\n ".format(tot_inf_actual, sft.get_val("time= ", line), tot_inf_expected, line)) elif "Extrapulmonary" in line: extrapulmonary_data_points += 1 if tot_inf_actual != 0: success = False outfile.write("BAD, found Extrapulmonary infection with total_infectiousness= {} at time= {}. " "Should be 0. \n".format(tot_inf_actual, sft.get_val("time= ", line))) outfile.write("Data points for each TBHIV infection state:\nLatent = {} \nPresymptomatic = {} " "\nSmear Negative = {} \nSmear Positive = {} \nExtrapulmonary = {} " "\n".format(latent_data_points, presymptomatic_data_points, smear_negative_data_points, smear_positive_data_points, extrapulmonary_data_points)) outfile.write("SUMMARY: Success={0}\n".format(success))
def application(report_file): sft.wait_for_done() # print( "Post-processing: " + report_file ) cdj = json.loads(open("config.json").read())["parameters"] start_time = cdj["Start_Time"] timestep = start_time lines_contact = [] lines_environment = [] contact_shedding_count = 0 contact_shedding_count_per_day = [] env_shedding_count = 0 env_shedding_count_per_day = [] with open("test.txt") as logfile: for line in logfile: # collect all lines of if re.search("Update\(\): Time:", line): # calculate time step timestep += 1 contact_shedding_count_per_day.append(contact_shedding_count) env_shedding_count_per_day.append(env_shedding_count) contact_shedding_count = env_shedding_count = 0 # reset counter elif re.search("depositing", line): # append time step and depositing line to lists ind_id = sft.get_val("Individual ", line) line = "TimeStep: " + str(timestep) + " ind_id: " + str(ind_id) + " " + line if re.search("contact", line): lines_contact.append(line) contact_shedding_count += 1 elif re.search("environment", line): lines_environment.append(line) env_shedding_count += 1 success = True with open(sft.sft_output_filename, "w") as report_file: if contact_shedding_count_per_day != env_shedding_count_per_day: success = False report_file.write("BAD: the # od individual shedding to contact and environment pool doesn't matches") sft.plot_data(contact_shedding_count_per_day, env_shedding_count_per_day, label1="Contact", label2="Environmental", title="Multi-route Shedding(count of individual shedding per day)", xlabel="Time", ylabel="# of individuals shedding into pool", category='shedding_multiroute_count', overlap=True, alpha=0.5) if not lines_contact or not lines_environment: success = False report_file.write("BAD: Found no data matching test case.\n") else: contact_shedding_per_day = [] env_shedding_per_day = [] for line in lines_contact: infectiousness2 = None ind_id = int(sft.get_val("ind_id: ", line)) timestep = sft.get_val("TimeStep: ", line) infectiousness1 = float(sft.get_val("depositing ", line)) for l in lines_environment: if "ind_id: " + str(ind_id) + " " in l and \ "TimeStep: " + str(timestep) + " " in l: infectiousness2 = float(sft.get_val("depositing ", l)) if infectiousness1 != infectiousness2: success = False report_file.write("BAD: Individual {0} is depositing {1} to route {2} and {3} to route {4} " "at time {5}.\n".format(ind_id, infectiousness1, "contact", infectiousness2, "environment", timestep)) # for plotting if ind_id == 1: contact_shedding_per_day.append(round(infectiousness1)) env_shedding_per_day.append(round(infectiousness2)) lines_environment.remove(l) break if infectiousness2 is None: success = False report_file.write("BAD: Individual {0} is not depositing to route environment at time {1}." "\n".format(ind_id, timestep)) sft.plot_data(contact_shedding_per_day, env_shedding_per_day,label1="Contact", label2="Environmental", title="Multi-route Shedding(Infectiousness of individual 1)", xlabel="Time", ylabel="Infectiousness", category='shedding_multiroute', overlap=True, alpha=0.5) if lines_environment: success = False report_file.write("BAD: {0} Individuals are not depositing to route contact while they are depositing " "to route contact.\n".format(len(lines_environment), timestep)) for l in lines_environment: report_file.write(l) report_file.write(sft.format_success_msg(success))
def application(report_file, debug=False): sft.wait_for_done() cdj = json.loads(open("config.json").read())["parameters"] tcer = float(cdj["Typhoid_Contact_Exposure_Rate"]) teer = float(cdj["Typhoid_Environmental_Exposure_Rate"]) num_exposures_contact = [] num_exposures_enviro = [] with open("test.txt") as logfile: for line in logfile: if "Exposing " in line and "route 'environment'" in line: # collect # of exposures for route contact num_exp_e = int(sft.get_val("num_exposures=", line)) num_exposures_enviro.append(num_exp_e) elif "Exposing " and "route 'contact'" in line: # collect # of exposures for route environment num_exp_c = int(sft.get_val("num_exposures=", line)) num_exposures_contact.append(num_exp_c) success = True with open(sft.sft_output_filename, "w") as report_file: # report_file.write("len1={0}, len2={1}, teer = {2}, tcer = {3}.\n".format(len(num_exposures_enviro), # len(num_exposures_contact), teer, tcer)) if not num_exposures_enviro: success = False report_file.write( "BAD: Found no individual exposed from route environment.\n") elif not sft.test_poisson(num_exposures_enviro, teer, report_file, 'environment'): success = False if not num_exposures_contact: success = False report_file.write( "BAD: Found no individual exposed from route contact.\n") elif not sft.test_poisson(num_exposures_contact, tcer, report_file, 'contact'): success = False report_file.write( "num_exposures_enviro = {0}, num_exposures_contact = {1}" "\n".format(len(num_exposures_enviro), len(num_exposures_contact))) # Write summary message to report file report_file.write(sft.format_success_msg(success)) with open("error_larger_than_tolerance_log.txt", "w") as my_file: sft.plot_poisson_probability( teer, num_exposures_enviro, my_file, 'environmental route', xlabel="Number of Exposures", ylabel="Probability", label2="Emod data", title="Poisson Probability Mass Funciton\n" "environmental route: Exposure_Rate = {}".format(teer)) sft.plot_poisson_probability( tcer, num_exposures_contact, my_file, 'contact route', xlabel="Number of Exposures", ylabel="Probability", label2="Emod data", title="Poisson Probability Mass Funciton\n" "contact route: Exposure_Rate = {}".format(tcer))
def application(report_file): sft.wait_for_done() # print( "Post-processing: " + report_file ) cdj = json.loads(open("config.json").read())["parameters"] ncdr = cdj["Node_Contagion_Decay_Rate"] start_time = cdj["Start_Time"] simulation_duration = int(cdj["Simulation_Duration"]) isj = json.loads(open("output/InsetChart.json").read())["Channels"] environmental_contagion_population = isj[ "Environmental Contagion Population"]["Data"] timestep = start_time lines = [] cum_all = [] cum = 0 Statpop = [] adding_cp_log = [] adding_cp = 0 shedding = 0 with open("test.txt") as logfile: for line in logfile: if "Update(): Time:" in line: # calculate time step timestep += 1 pop = float(sft.get_val("StatPop: ", line)) Statpop.append(pop) # append the accumulated shedding and reset the counter at the end of each time step. # data for timestep 1 is stored in cum_all[1] cum_all.append(cum) # environmental cp decay cum *= 1.0 - ncdr adding_cp_log.append(adding_cp) # resetting shedding and adding_cp variables shedding = 0 adding_cp = 0 elif "[MultiRouteTransmissionGroups] Adding " in line and "route:1" in line: # append time step and total shedding line to lists # line = "TimeStep: " + str(timestep) + " " + line # lines.append(line) line = "TimeStep: " + str(timestep) + " " + line lines.append(line) adding_cp = float(sft.get_val("Adding ", line)) elif "Exposing " in line and "route 'environment'" in line: # append time step and Exposing line to lists line = "TimeStep: " + str(timestep) + " " + line lines.append(line) elif "depositing" in line and "route environment" in line: # get shedding of contact route and add it to accumulated shedding shedding = float(sft.get_val("depositing ", line)) cum += shedding line = "TimeStep: " + str(timestep) + " " + line lines.append(line) # elif "scaled by " in line and "route:1" in line: # # get population # if timestep > 0: # # start collecting population at time 1 # pop = float(sft.get_val("scaled by ", line)) # Statpop.append(pop) with open("DEBUG_filtered_from_StdOut.txt", "w") as file: file.write("".join(lines)) expected_cp_insetchart = [] cp_stdout_dict = {} success = True with open(sft.sft_output_filename, "w") as report_file: if len(lines) == 0: success = False report_file.write("Found no data matching test case.\n") else: if len(cum_all) != simulation_duration or len( Statpop) != simulation_duration: report_file.write( "WARNING: the test.txt file is incomplete. len(cum_all)={0}, len(Statpop)={1}," " expected {2}.\n".format(len(cum_all), len(Statpop), simulation_duration)) for line in lines: if re.search("Exposing", line): environment = float(sft.get_val("environment=", line)) timestep = int(sft.get_val("TimeStep: ", line)) expected_cp = cum_all[timestep - start_time] / Statpop[timestep - start_time - 1] if timestep not in cp_stdout_dict: cp_stdout_dict[timestep] = [environment, expected_cp] if math.fabs(environment - expected_cp) > 1e-2 * expected_cp: success = False ind_id = int(sft.get_val("inividual ", line)) report_file.write( "BAD: at time {0}, individuals are exposed on route environment with contagion " "population = {1} StatPop = {2}, expected {3}." "\n".format(timestep, environment, Statpop[timestep - start_time - 1], expected_cp)) pre_cum = 0 for timestep in range(len(cum_all) - 1): # report_file.write("timestep={0}, StatPop = {1}, cum_all= {2}, adding_cp_log = {3}. # \n".format(x,Statpop[x+1],cum_all[x], adding_cp_log[x])) # the InsetChart is missing the first time step, so we're taking it out from the log. #2890 # cur_cum = cum_all[timestep + 1] cur_cum = cum_all[timestep] # comment out these code since we are no longer logging the accumulated shedding in StdOut file. # adding_cp = cur_cum - pre_cum * (1.0 - ncdr) # # pass cur_cum to pre_cum for next time step # pre_cum = cur_cum # if math.fabs(adding_cp - adding_cp_log[x]) > 1e-1: # success = False # report_file.write( # "BAD: At time {0}, the accumulated shedding is {1} from StdOut, expected {2}." # "\n".format(timestep, adding_cp_log[x], adding_cp)) expected_cp = cur_cum / Statpop[timestep] expected_cp_insetchart.append(expected_cp) if math.fabs(expected_cp - environmental_contagion_population[timestep] ) > 1e-2 * expected_cp: success = False report_file.write( "BAD: At time {0}, the environmental contagion population is {1} from InsetChart.json, " "expected {2}.\n".format( timestep, environmental_contagion_population[timestep], expected_cp)) report_file.write(sft.format_success_msg(success)) sft.plot_data( environmental_contagion_population, expected_cp_insetchart, label1="InsetChart", label2="expected data", title= "Environmental Contagion Population\n InsetChart vs. expected", xlabel="Day", ylabel="Environmental Contagion Population", category='environmental_contagion_population_InsetChart', alpha=0.5, overlap=True) sft.plot_data(np.array(list(cp_stdout_dict.values()))[:, 0], np.array(list(cp_stdout_dict.values()))[:, 1], label1="StdOut", label2="expected data", title="Exposure(Contagion)\n StdOut vs. expected", xlabel="Day", ylabel="Contagion", category='exposure_StdOut', alpha=0.5, overlap=True) cp_list_for_plot = [] for key in sorted(cp_stdout_dict): cp_list_for_plot.append([ cp_stdout_dict[key][0], environmental_contagion_population[key - 1] ]) sft.plot_data( np.array(cp_list_for_plot)[:, 1][1:], np.array(cp_list_for_plot)[:, 0], label1="InsetChart", label2="StdOut", title="Environmental Contagion Population\n InsetChart vs. StdOut", xlabel="Day", ylabel="Contagion", category='environmental_contagion_population', alpha=0.5, overlap=True)
def parse_stdout_file(start_time=1, stdout_filename="test.txt", filtered_filename="filtered_lines.txt", debug=False): """ Goes through the file and collects information on daily new acute infections (to verify with InsetChart.json), acute infection to treatment times and exceptions (if person goes acute while already acute, or treated after being recovered,etc) :param start_time: first time step from config :param stdout_filename: file to parse (test.txt) :param filtered_filename: file to parse to (only happens in debug) :param debug: whether or not data gets saved to to the external file :return: acute_to_treatment_times, acute_infections_daily, exceptions """ with open(stdout_filename) as infile: time_step = start_time exceptions = [] filtered_lines = [] acute_to_treatment_times = [] acute_infections = {} acute_infections_daily = [] acute_infections_count_daily = 0 # no one dies in current application from the infection, we will have to change some things if people start # dying. Counting up time steps between individuals going acute to treatment, throwing out individuals # who self-cure or go chronic for line in infile: if UPDATE_TIME in line: filtered_lines.append(line) # calculate time step, count add daily infection count, reset daily infection count time_step += 1 acute_infections_daily.append(acute_infections_count_daily) acute_infections_count_daily = 0 elif TO_ACUTE in line: filtered_lines.append(line) acute_infections_count_daily += 1 ind_id = int(sft.get_val(IND_ID_1, line)) if ind_id not in acute_infections: acute_infections[ind_id] = time_step else: exceptions.append( "Individual {} at time {} just became acute while already being " "acute\n".format(ind_id, time_step)) elif TREATMENT in line: filtered_lines.append(line) # when person gets treated, calculate the time from becoming acute to being treated ind_id = int(sft.get_val(IND_ID_2, line)) if ind_id in acute_infections: acute_to_treatment_times.append( time_step - acute_infections.get(ind_id)) else: exceptions.append( "Individual {} at time {} got treatment without being acute\n" .format(ind_id, time_step)) elif RECOVERED in line: filtered_lines.append(line) # when person is recovered, kick them out of acute_infections dictionary, stop keeping track of them ind_id = int(sft.get_val(IND_ID_3, line)) if ind_id in acute_infections: del acute_infections[ind_id] else: exceptions.append( "Individual {} at time {} recovered from acute without being acute\n" .format(ind_id, time_step)) elif CHRONIC in line: filtered_lines.append(line) # when person goes chronic, kick them out of acute_infections dictionary, stop keeping track of them ind_id = int(sft.get_val(IND_ID_3, line)) if ind_id in acute_infections: del acute_infections[ind_id] else: exceptions.append( "Individual {} at time {} just went chronic from acute without " "being acute\n".format(ind_id, time_step)) if debug: with open(filtered_filename, "w") as outfile: outfile.writelines(filtered_lines) return [acute_to_treatment_times, acute_infections_daily, exceptions]