def run_single_analyzer(exp_id, analyzer, savefile_prefix=""): SetupParser.default_block = 'HPC' SetupParser.init() am = AnalyzeManager() am.add_analyzer(analyzer()) exp = retrieve_experiment(exp_id) am.add_experiment(exp) am.analyze() df_return = am.analyzers[0].results df_return.to_csv("{}_{}.csv".format(savefile_prefix, exp_id), index=False) return df_return
def plot_inset_diagnostics(experiment_list, channels=default_channels, working_dir=".", filename="output/InsetChart.json", **kwargs): from simtools.Analysis.AnalyzeManager import AnalyzeManager am = AnalyzeManager() am.add_analyzer( inset_channel_plotter(channels, working_dir=working_dir, filename=filename, **kwargs)) for expt in experiment_list: am.add_experiment(expt) am.analyze()
def run_experiment(configbuilder, experiment_name, experiment_builder, analyzers): run_sim_args = { 'config_builder': configbuilder, 'exp_name': experiment_name, 'exp_builder': experiment_builder } if not SetupParser.initialized: SetupParser.init('HPC') exp_manager = ExperimentManagerFactory.init() exp_manager.run_simulations(**run_sim_args) exp_manager.wait_for_finished(verbose=True) assert (exp_manager.succeeded()) am = AnalyzeManager(exp_manager.experiment) for a in analyzers: am.add_analyzer(a) am.analyze()
def run_analyzers(exp_id, analyzers, savefile_prefix=""): def _remove_duplicate_columns(df): columns_to_keep = [] for c in df.columns: if "_duplicated" not in c: columns_to_keep.append(c) return df[columns_to_keep] SetupParser.default_block = 'HPC' SetupParser.init() am = AnalyzeManager() for a in analyzers: am.add_analyzer(a()) exp = retrieve_experiment(exp_id) am.add_experiment(exp) am.analyze() if len(analyzers) == 1: df_return = am.analyzers[0].results elif len(analyzers) > 1: df_list = [x.results for x in am.analyzers] df_return = pd.merge(df_list[0], df_list[1], on="sim_id", suffixes=["", "_duplicated"]) # Drop duplicated columns # for c in df_result.columns: # if "_duplicated" in c: # df_result.drop(c, inplace=True) df_return = _remove_duplicate_columns(df_return) else: raise ValueError df_return.to_csv("{}_{}.csv".format(savefile_prefix, exp_id), index=False) return df_return
plt.plot(data, label=label, c=c) else: plt.plot(data, label=label) plt.xlabel("Simulation Time") plt.ylabel(self.channel) plt.legend() plt.show() if __name__ == "__main__": from simtools.Analysis.AnalyzeManager import AnalyzeManager am = AnalyzeManager() # am.add_analyzer(basic_inset_channel_plotter("True Prevalence")) am.add_analyzer( inset_channel_plotter("True Prevalence", color_by_expt=True, label_by_expt=True, label_dict={ "7e3073b4-d9f1-e811-a2bd-c4346bcb1555": "full campaign", "a2e981fe-d9f1-e811-a2bd-c4346bcb1555": "no 2011 bednets" }, ref_date="2001-01-01")) am.add_experiment("7e3073b4-d9f1-e811-a2bd-c4346bcb1555") am.add_experiment("a2e981fe-d9f1-e811-a2bd-c4346bcb1555") am.analyze()
# Write special serialized_files_sim_map for runs that need it for c in range(56): catch = catch_list[c] c_folder = "C:/Users/jsuresh/Dropbox (IDM)/Malaria Team Folder/projects/zambia_gridded_sims/kariba_gridded_sims/calibs/{}".format( catch) sim_map_filename = os.path.join(c_folder, "serialized_files_sim_map.csv") # See if you can find an experiment named CATCH_project_climatefix try: name_try = "{}_project_climatefix".format(catch) proj_exp = DataStore.get_most_recent_experiment( id_or_name=name_try) # If you can find the experiment, find the corresponding parent simulation and output its sim map if proj_exp: name_try = "{}_serialize".format(catch) orig_exp = DataStore.get_most_recent_experiment( id_or_name=name_try) am = AnalyzeManager() am.add_experiment(orig_exp) am.add_analyzer( SimulationDirectoryMapAnalyzer(save_file=sim_map_filename)) am.analyze() print("Wrote serialized_files_sim_map for {}".format(catch)) except: pass
# This block will be used unless overridden on the command-line SetupParser.default_block = 'HPC' cb = DTKConfigBuilder.from_defaults('VECTOR_SIM') configure_site(cb, 'Namawala') cb.set_param('Simulation_Duration',365) analyzers = (TimeseriesAnalyzer(), VectorSpeciesAnalyzer()) builder = GenericSweepBuilder.from_dict({'Run_Number': range(5)}) run_sim_args = { 'exp_name': 'testrunandanalyze', 'exp_builder': builder, 'config_builder':cb } if __name__ == "__main__": SetupParser.init(selected_block=SetupParser.default_block) exp_manager = ExperimentManagerFactory.from_cb(config_builder=cb) exp_manager.run_simulations(**run_sim_args) exp_manager.wait_for_finished(verbose=True) am = AnalyzeManager(exp_manager.experiment) for a in analyzers: am.add_analyzer(a) am.analyze()
finished = all([em.finished() for em in experiments]) # Create a new AnalyzeManager and add experiment and analyzer am = AnalyzeManager(verbose=False) for em in experiments: am.add_experiment(em.experiment) analyzer = DownloadAnalyzerTPI( filenames=['output\\ReportHIVByAgeAndGender.csv'], TPI_tag="TPI", ignore_TPI=False, REP_tag="TPI", ignore_REP=True, output_path=output_directory) am.add_analyzer(analyzer) # Make sure we refresh our set of experiments for e in experiments: e.refresh_experiment() COMPS_login(SetupParser.get("server_endpoint")) am.analyze() # If we are not done we wait for 30 sec, if we are done we leave if not finished: print("Waiting 30 seconds") time.sleep(30) else: break
cb = CMSConfigBuilder.from_files(model_file='inputs/models/simplemodel.emodl', config_file='inputs/models/simplemodel.cfg') # If the base collection containing CMS exists, use it # If not, use the local if SetupParser.default_block == "HPC": try: cb.set_collection_id('CMS 0.82 Pre-release') except SimulationAssets.InvalidCollection: cb.set_experiment_executable('inputs/compartments/compartments.exe') cb.set_dll_root('inputs/compartments') else: cb.set_experiment_executable('inputs/compartments/compartments.exe') cb.set_dll_root('inputs/compartments') run_sim_args = {"config_builder": cb, "exp_name": "First CMS run"} if __name__ == "__main__": SetupParser.init() em = ExperimentManagerFactory.from_cb(run_sim_args["config_builder"]) em.run_simulations(exp_name=run_sim_args["exp_name"]) # Wait for the simulation to complete em.wait_for_finished(verbose=True) # Analyze am = AnalyzeManager(exp_list='latest') am.add_analyzer(SimpleCMSAnalyzer()) am.analyze()
data_list.append(all_data[sim]) return pd.concat(data_list) def finalize(self, all_data): sim_data_full = self.combine(all_data) # print("all_data ",all_data) # print("sim_data_full ", sim_data_full) if self.save_file: sim_data_full.to_csv(self.save_file, index=False) return sim_data_full if __name__ == "__main__": SetupParser.default_block = 'HPC' SetupParser.init() exp_id = sys.argv[1] # start_time_step = int(sys.argv[2]) am = AnalyzeManager() am.add_analyzer( SaveEndpoint( save_file="endpoints_{}.csv".format(exp_id), # start_time_step=start_time_step, include_counter_report=True, output_filename="ReportMalariaFilteredFinal_Year.json")) exp = retrieve_experiment(exp_id) am.add_experiment(exp) am.analyze()
.format(rounds_so_far, len(sims_so_far["id"]))) available_sims = sims_so_far.copy() print("Samples : {}".format(available_sims["sample"])) if len(available_sims) > 0: am = AnalyzeManager() for sim_id in available_sims["id"]: am.add_simulation(retrieve_simulation(sim_id)) am.add_analyzer( comparison_channel_plotter( "Blood Smear Parasite Prevalence", filenames=['output/ReportMalariaFilteredCatchment.json'], reference_data=prev_ref_data, ref_date=ref_date, legend=False, working_dir=catch_folder, save_type="other", save_name="excl_prev_test")) am.analyze() plt.figure() # Set plot range to 2017: for s in available_sims["sample"]: df = inc_comp_df[inc_comp_df["sample"] == s] plt.plot(np.array(df["cases_sim"])) plt.plot(np.array(df["cases_ref"]), color='black',
if len(sm_mini) > 0: if __name__ == "__main__": am = AnalyzeManager() for sim_id in sm_mini["id"]: am.add_simulation(retrieve_simulation(sim_id)) am.add_analyzer( comparison_channel_plotter( "Blood Smear Parasite Prevalence", filenames=[ 'output/ReportMalariaFilteredCatchment.json' ], reference_data=prev_ref_data, ref_date=ref_date, legend=False, working_dir='.', save_type="other", save_name=os.path.join( c_folder, "prev_dist_thresh_{}_inc_{}.png".format(i, j)))) am.analyze() pretty_plot_incidence( inc_comp_df, list(sm_mini["__sample_index__"]), savefile=os.path.join( c_folder,
# An example: if __name__ == "__main__": from simtools.Analysis.AnalyzeManager import AnalyzeManager am = AnalyzeManager() # am.add_analyzer(basic_inset_channel_plotter("True Prevalence")) diagnostic_channels = [ "Daily Bites per Human", "Daily EIR", "Adult Vectors", "Blood Smear Parasite Prevalence" ] am.add_analyzer( inset_channel_plotter(diagnostic_channels, color_by_expt=True, label_by_expt=True, label_dict={ "7e3073b4-d9f1-e811-a2bd-c4346bcb1555": "full campaign", "a2e981fe-d9f1-e811-a2bd-c4346bcb1555": "no 2011 bednets" }, ref_date="2001-01-01", figsize=(10, 4), plot_xlim=convert_date_list_to_mdate( ["2010-01-01", "2019-01-01"]), plot_show=False)) am.add_experiment("7e3073b4-d9f1-e811-a2bd-c4346bcb1555") am.add_experiment("a2e981fe-d9f1-e811-a2bd-c4346bcb1555") am.analyze()
'f8394f29-b6e2-e811-a2bd-c4346bcb1555', 'cd70a86e-b5e2-e811-a2bd-c4346bcb1555', 'c47d78b3-b4e2-e811-a2bd-c4346bcb1555', 'c6cbf126-59dd-e811-a2bd-c4346bcb1555', 'c9eef583-58dd-e811-a2bd-c4346bcb1555', '50a088ee-57dd-e811-a2bd-c4346bcb1555' ] ref_exp = load_experiments_from_file() for exp in exp_list: tmp = get_experiment_by_id( exp, query_criteria=QueryCriteria().select_children(["tags"])) if (exp in ref_exp) or (any( [sim.state.value != 6 for sim in tmp.get_simulations()])): continue with open(os.path.join('Experiments', 'experiment_metadata.json'), 'r', encoding='utf8') as jsonfile: exp_metadata = json.load(jsonfile) exp_metadata[exp] = tmp.tags with open(os.path.join('Experiments', 'experiment_metadata.json'), 'w', encoding='utf8') as jsonfile: json.dump(exp_metadata, jsonfile, cls=NumpyEncoder, indent=3) am = AnalyzeManager(exp) am.add_analyzer(Output2MatlabAnalyzer()) am.analyze()