def test_declination_sensitivity(self): logging.info("Testing 'fixed_weight' MinimisationHandler class") for i, e_pdf_dict in enumerate(energy_pdfs): llh_dict = { "llh_name": "fixed_energy", "llh_sig_time_pdf": { "time_pdf_name": "steady" }, "llh_bkg_time_pdf": { "time_pdf_name": "steady", }, "llh_energy_pdf": e_pdf_dict } unblind_dict = { "name": "tests/test_energy_pdfs/", "mh_name": "fixed_weights", "dataset": icecube_ps_3_year.get_seasons('IC79-2010', 'IC86-2011'), "catalogue": catalogue, "llh_dict": llh_dict, } ub = create_unblinder(unblind_dict) key = [x for x in ub.res_dict.keys() if x != "TS"][0] res = ub.res_dict[key] logging.info("Best fit values {0}".format(list(res["x"]))) logging.info("Reference best fit {0}".format(true_parameters[i])) for j, x in enumerate(list(res["x"])): self.assertAlmostEqual(x, true_parameters[i][j], delta=0.1)
def test_declination_sensitivity(self): logging.info("Testing 'large_catalogue' MinimisationHandler class " "with {0} sources and IC40 data".format(n_sources)) # Test stacking unblind_dict = { "name": "test/test_large_catalogue/", "mh_name": "large_catalogue", "dataset": icecube_ps_3_year.get_seasons("IC79-2010"), "catalogue": catalogue, "llh_dict": llh_dict, "inj_dict": {} } ub = create_unblinder(unblind_dict) key = [x for x in ub.res_dict.keys() if x != "TS"][0] res = ub.res_dict[key] logging.info("Best fit values {0}".format(list(res["x"]))) logging.info("Reference best fit {0}".format(true_parameters[0])) for i, x in enumerate(res["x"]): self.assertAlmostEqual(x, true_parameters[0][i], delta=0.1)
def test_declination_sensitivity(self): logging.info("Testing 'standard' LLH class") # Test three declinations for j, sindec in enumerate(sindecs): unblind_dict = { "name": "tests/test_llh_standard/", "mh_name": "fixed_weights", "dataset": icecube_ps_3_year.get_seasons("IC86-2011"), "catalogue": ps_catalogue_name(sindec), "llh_dict": llh_dict, } ub = create_unblinder(unblind_dict) key = [x for x in ub.res_dict.keys() if x != "TS"][0] res = ub.res_dict[key] logging.info("Best fit values {0}".format(list(res["x"]))) logging.info("Reference best fit {0}".format(true_parameters[j])) for i, x in enumerate(res["x"]): self.assertAlmostEqual(x, true_parameters[j][i], delta=0.1)
def test_spatial(self): logging.info("Testing 'spatial' LLH class") ub = create_unblinder(unblind_dict) key = [x for x in ub.res_dict.keys() if x != "TS"][0] res = ub.res_dict[key] logging.info("Best fit values {0}".format(list(res["x"]))) logging.info("Reference best fit {0}".format(true_parameters)) self.assertAlmostEqual(list(res["x"])[0], true_parameters, delta=0.1)
def test_custom_dataset(self): logging.info("Testing custom_dataset util function.") llh_dict = { "llh_name": "standard", "llh_sig_time_pdf": { "time_pdf_name": "box", "pre_window": 0., "post_window": 100. }, "llh_bkg_time_pdf": { "time_pdf_name": "steady", }, "llh_energy_pdf": { "energy_pdf_name": "power_law" } } dataset = custom_dataset(icecube_ps_3_year, load_catalogue(catalogue), llh_dict["llh_sig_time_pdf"]) keys = sorted(list(dataset.keys())) self.assertEqual(keys, true_keys) # Test three declinations unblind_dict = { "name": "test_custom_dataset", "mh_name": "fixed_weights", "dataset": custom_dataset(icecube_ps_3_year, load_catalogue(catalogue), llh_dict["llh_sig_time_pdf"]), "catalogue": catalogue, "llh_dict": llh_dict, } ub = create_unblinder(unblind_dict) key = [x for x in ub.res_dict.keys() if x != "TS"][0] res = ub.res_dict[key] for j, x in enumerate(res["x"]): self.assertAlmostEqual(x, true_parameters[j], delta=0.1) logging.info("Best fit values {0}".format(list(res["x"]))) logging.info("Reference best fit {0}".format(true_parameters))
def test_declination_sensitivity(self): logging.info("Testing 'fit_weight' MinimisationHandler class") mh_name = "fit_weights" # Test three declinations unblind_dict = { "name": "tests/test_mh_fit_weights", "mh_name": mh_name, "dataset": icecube_ps_3_year.get_seasons("IC86-2011"), "catalogue": catalogue, "llh_dict": llh_dict, } ub = create_unblinder(unblind_dict) key = [x for x in ub.res_dict.keys() if x != "TS"][0] res = ub.res_dict[key] logging.info("Best fit values {0}".format(list(res["x"]))) logging.info("Reference best fit {0}".format(true_parameters)) for i, x in enumerate(res["x"]): self.assertAlmostEqual(x, true_parameters[i], delta=0.1) inj_dict = { "injection_sig_time_pdf": {"time_pdf_name": "steady"}, "injection_bkg_time_pdf": { "time_pdf_name": "steady", }, "injection_energy_pdf": {"energy_pdf_name": "power_law", "gamma": 2.0}, } mh_dict = dict(unblind_dict) mh_dict["inj_dict"] = inj_dict mh_dict["n_trials"] = 1.0 mh_dict["n_steps"] = 3.0 mh_dict["scale"] = 5.0 mh = MinimisationHandler.create(mh_dict) res = mh.simulate_and_run(5.0) analyse(mh_dict, cluster=False) mh.corner_likelihood_scan( save=True, res_dict=res, )
def test_flare(self): logging.info("Testing 'flare' MinimisationHandler class") ub = create_unblinder(unblind_dict, full_plots=True) res = [x for x in ub.res_dict["Parameters"].values()] logging.info("Best fit values {0}".format(list(res))) logging.info("Reference best fit {0}".format(true_parameters)) for i, x in enumerate(res): if i < 2: self.assertAlmostEqual(x / true_parameters[i], 1.0, places=1) else: self.assertEqual(x, true_parameters[i]) inj_dict = { "injection_sig_time_pdf": { "time_pdf_name": "steady" }, "injection_bkg_time_pdf": { "time_pdf_name": "steady", }, "injection_energy_pdf": { "energy_pdf_name": "power_law", "gamma": 2.0 }, } mh_dict = dict(unblind_dict) mh_dict["inj_dict"] = inj_dict mh_dict["n_trials"] = 1.0 mh_dict["n_steps"] = 3.0 mh_dict["scale"] = 5.0 mh = MinimisationHandler.create(mh_dict) res = mh.simulate_and_run(5.0) analyse(mh_dict, cluster=False)
for gamma_index in gammas: res = dict() nr_srcs = int(nr_brightest_sources[0]) cat_path = agn_subset_catalogue(cat_type, method, nr_srcs) catalogue = load_catalogue(cat_path) cat = np.load(cat_path) name = generate_name(unique_key, nr_srcs, gamma_index) bkg_ts = bkg_ts_base_name(unique_key, nr_srcs) injection_time = llh_time injection_energy = dict(llh_energy) injection_energy["gamma"] = gamma_index inj_kwargs = { "injection_energy_pdf": injection_energy, "injection_sig_time_pdf": injection_time, } unblind_dict = { "name": name, "mh_name": "large_catalogue", "dataset": diffuse_8_year.get_seasons(), "catalogue": cat_path, "llh_dict": llh_dict, "background_ts": bkg_ts, } ub = create_unblinder(unblind_dict, mock_unblind=True, full_plots=False)
for pdf_type in sn_times[cat]: llh_times = sn_time_pdfs(cat, pdf_type=pdf_type) name = f"{name_root}/{pdf_type}/{cat}" bkg_ts = f"{bkg_ts_root}/{pdf_type}/{cat}" for llh_time in llh_times: time = (llh_time["decay_time"] if "decay" in pdf_type else llh_time["pre_window"] + llh_time["post_window"]) unblind_llh = { "llh_name": "standard", "llh_energy_pdf": llh_energy, "llh_sig_time_pdf": llh_time, } unblind_dict = { "name": name, "ts_type": "Fit Weights", "mh_name": "fit_weights", "dataset": custom_dataset(ps_v002_p01, catalogue, llh_time), "catalogue": cat_path, "llh_dict": unblind_llh, "background_ts": f"{bkg_ts}/{time}/", } ub = create_unblinder(unblind_dict, mock_unblind=True)
name, "mh_name": "fit_weights", "dataset": custom_dataset(txs_sample_v1, catalogue, llh_dict["llh_sig_time_pdf"]), "catalogue": cat_path, "llh_dict": llh_dict, "background_ts": bkg_ts } # ub = create_unblinder(unblind_dict, mock_unblind=False) ub = create_unblinder(unblind_dict, mock_unblind=False, disable_warning=True) r = ub.res_dict print(r) ns = np.zeros(len(catalogue)) for x, val in r["Parameters"].items(): if "n_s" in x: mask = np.array( [str(y) in str(x) for y in catalogue["source_name"]]) ns[mask] = float(val) print(f"nstot: {np.sum(ns)}") res.append((cat, ub.ts))
} res_dict = dict() for cat in sn_cats: name = name_root + cat + "/" bkg_ts = bkg_ts_root + cat + "/" cat_path = sn_catalogue_name(cat) catalogue = np.load(cat_path) llh_times = sn_time_pdfs(cat) for llh_time in llh_times: unblind_llh = { "llh_name": "standard", "llh_energy_pdf": llh_energy, "llh_time_pdf": llh_time, } unblind_dict = { "name": name, "mh_name": "fit_weights", "dataset": custom_dataset(ps_v002_p01, catalogue, llh_time), "catalogue": cat_path, "llh_dict": unblind_llh, "background_ts": bkg_ts, } ub = create_unblinder(unblind_dict, mock_unblind=False)
def run_background_pvalue_trials(seed=None, ntrials=1): """ Run background trials and calculate the p-values :param seed: int, seed for dataset simulation :param ntrials: int, number of trials to run :return: dictionary containing the trial results """ # setting the random seed ensures that the dataset produced by the Unblinder # will always produce the same dataset rng = np.random.default_rng(seed=seed) # generating an array of seeds that will be used if more than one trial is run seeds = rng.integers(0, 2**30, ntrials) pdf_full_res = dict() for pdf_type in ["box", "decay"]: # base name raw = f"{base_raw}/{pdf_type}/" # set up emtpy dictionary to store the minimizer information in full_res = dict() # loop over SN catalogues use_cats = sn_cats if pdf_type == "box" else ["IIn", "IIP"] for cat in use_cats: name = raw + cat + "/" # set up empty results dictionary for this catalogue cat_res = dict() # get the time pdfs for this catalogue time_pdfs = sn_time_pdfs(cat, pdf_type=pdf_type) # Loop over time PDFs for llh_time in time_pdfs: # set up an empty results array for this time pdf time_res = dict() logging.debug(f"time_pdf is {llh_time}") if pdf_type == "box": time_key = str(llh_time["post_window"] + llh_time["pre_window"]) # pdf_time = float(time_key) if llh_time['pre_window'] == 0 else - float(time_key) else: time_key = str(llh_time["decay_time"]) time_name = name + time_key + "/" # Loop over spectral indices for gamma in gammas: # load the background trials if (pdf_type == "decay") and (gamma == 2.0): gamma_str = "2" else: gamma_str = str(gamma) full_name = time_name + gamma_str + "/" mh_dict_path = analysis_pickle_path(name=full_name) logger.debug( f"loading analysis pickle from {mh_dict_path}") with open(mh_dict_path, "rb") as f: mh_dict = pickle.load(f) unblind_dict = dict(mh_dict) unblind_dict["name"] += "_create_backround_pvalue_distr" unblind_dict["background_ts"] = mh_dict["name"] ub = create_unblinder( unblind_dict, mock_unblind=True, disable_warning=True, seed=seeds[0], ) bkg_ts_array = ub.bkg_ts_array p_values = [ub.raw_pre_trial_pvalue] logger.info(f"Raw p-value is {ub.raw_pre_trial_pvalue}") for s in seeds[1:]: int_seed = s logger.debug(f"seed is {int_seed}") res_dict = ub.simulate_and_run(0, int_seed) ts = res_dict["TS"] raw_p_value = len(bkg_ts_array[ bkg_ts_array > ts]) / len(bkg_ts_array) logger.info(f"Raw p-value is {raw_p_value}") p_values.append(raw_p_value) time_res[gamma] = p_values cat_res[time_key] = time_res full_res[cat] = cat_res pdf_full_res[pdf_type] = full_res return pdf_full_res