def find_sensitivity(self):
"""Uses the results of the background trials to find the median TS
value, determining the sensitivity threshold. This sensitivity is
not necessarily zero, for example with negative n_s, fitting of
weights or the flare search method.
"""
try:
bkg_dict = self.results[scale_shortener(0.0)]
except KeyError:
logger.error("No key equal to '0'")
return
bkg_ts = bkg_dict["TS"]
bkg_median = np.median(bkg_ts)
self.bkg_median = bkg_median
savepath = os.path.join(self.plot_dir, "sensitivity.pdf")
self.find_overfluctuations(bkg_median, savepath)
# self.sensitivity_fit(savepath, bkg_median)
# self.sensitivity, self.extrapolated_sens, self.sensitivity_err = self.find_overfluctuations(
# bkg_median, savepath, bkg_median
# )
msg = ""
if self.extrapolated_sens:
msg = "EXTRAPOLATED "
logger.info("{0}Sensitivity is {1:.3g}".format(msg, self.sensitivity))
def find_overfluctuations(self, ts_val, savepath=None):
"""Uses the values of injection trials to fit an 1-exponential decay
function to the overfluctuations, allowing for calculation of the
sensitivity. Where the injected flux was not sufficient to reach the
sensitivity, extrapolation will be used instead of interpolation,
but this will obviously have larger associated errors. If
extrapolation is used, self.extrapolated_sens is set to true. In
either case, a plot of the overfluctuations as a function of the
injected signal will be made.
"""
x = sorted(self.results.keys())
x_acc = []
y = []
x = [scale_shortener(i) for i in sorted([float(j) for j in x])]
yerr = []
for scale in x:
ts_array = np.array(self.results[scale]["TS"])
frac = float(len(ts_array[ts_array > ts_val])) / (float(
len(ts_array)))
logger.info(
"Fraction of overfluctuations is {0:.2f} above {1:.2f} (N_trials={2}) (Scale={3})"
.format(frac, ts_val, len(ts_array), scale))
if scale == scale_shortener(0.0):
self.frac_over = frac
if len(ts_array) > 1:
y.append(frac)
x_acc.append(float(scale))
yerr.append(1.0 / np.sqrt(float(len(ts_array))))
self.make_plots(scale)
if len(np.where(np.array(y) < 0.95)[0]) < 2:
raise OverfluctuationError(
f"Not enough points with overfluctuations under 95%, lower injection scale!"
)
x = np.array(x_acc)
self.overfluctuations[ts_val] = x, y, yerr
return self.sensitivity_fit(savepath, ts_val)
disc_dict = dict()
for (config, mh_list) in res_dict.items():
sens = []
med_biases = []
mean_biases = []
disc = []
for mh_dict in mh_list:
rh = ResultsHandler(mh_dict)
max_scale = scale_shortener(
max([float(x) for x in list(rh.results.keys())]))
sens.append(rh.sensitivity)
disc.append(rh.disc_potential)
fit = rh.results[max_scale]["Parameters"]["n_s"]
inj = rh.inj[max_scale]["n_s"]
med_bias = np.median(fit) / inj
med_biases.append(med_bias)
mean_biases.append(np.mean(fit) / inj)
# ax1.plot(sin_decs, sens, label=config)
sens_dict[config] = np.array(sens)
med_bias_dict[config] = med_biases
disc_dict[config] = np.array(disc)
mean_bias_dict[config] = mean_biases
gamma_str = "2"
else:
gamma_str = str(gamma)
sens_name = (
f"{base_raw}/{pdf_type}/{cat}/{time_key}/{gamma_str}"
)
sens_mh_dict = dict(mh_dict)
sens_mh_dict["name"] = sens_name
logging.info(
"----- loading sensitivity results -------")
sens_rh = ResultsHandler(sens_mh_dict,
do_disc=False,
do_sens=False)
# sens_rh = ExperimentalResultHandler(sens_mh_dict, do_sens=False, do_disc=False)
bkg_res = sens_rh.results[scale_shortener(0.0)]
# load the signal trials
# rh = ResultsHandler(mh_dict, do_disc=False, do_sens=False)
try:
logging.info(
"----------- loading energy range calculations ------------"
)
# rh = ExperimentalResultHandler(mh_dict, do_sens=False, do_disc=False)
rh = ResultsHandler(mh_dict,
do_sens=False,
do_disc=False)
# insert the background trials
logging.debug(
"setting background TS distribution from sensitivity results"
def plot_bias(self):
x = sorted(self.results.keys())
raw_x = [scale_shortener(i) for i in sorted([float(j) for j in x])]
base_x = [k_to_flux(float(j)) for j in raw_x]
base_x_label = r"$\Phi_{1GeV}$ (GeV$^{-1}$ cm$^{-2}$)"
for i, param in enumerate(self.param_names):
try:
plt.figure()
ax = plt.subplot(111)
meds = []
ulims = []
llims = []
trues = []
for scale in raw_x:
vals = self.results[scale]["Parameters"][param]
if self.bias_error == "std":
med = np.median(vals)
meds.append(med)
sig = np.std(vals)
ulims.append(med + sig)
llims.append(med - sig)
elif self.bias_error == "ci90":
med, llim, ulim = np.quantile(vals, [0.5, 0.05, 0.95])
meds.append(med)
llims.append(llim)
ulims.append(ulim)
else:
raise ValueError(
f"Invalid value {self.bias_error} for bias_error!")
true = self.inj[scale][param]
trues.append(true)
do_ns_scale = False
if "n_s" in param:
x = trues
x_label = r"$n_{injected}$" + param.replace("n_s", "")
else:
x = base_x
x_label = base_x_label
# decide wether to plot a second x axis on the top axis indicating the number of injected
# neutrinos instead of the flux
if "gamma" in param:
if not isinstance(self.flux_to_ns, type(None)):
do_ns_scale = True
ns_scale = ns_scale_label = None
if do_ns_scale:
ns_scale = self.flux_to_ns * max(base_x)
ns_scale_label = "Number of neutrinos"
plt.scatter(x, meds, color="orange")
plt.plot(x, meds, color="black")
plt.plot(x, trues, linestyle="--", color="red")
plt.fill_between(x, ulims, llims, alpha=0.5, color="orange")
try:
ax.set_xlim(left=0.0, right=max(x))
if min(trues) == 0.0:
ax.set_ylim(bottom=0.0)
if do_ns_scale:
ax2 = ax.twiny()
ax2.grid(0)
ax2.set_xlim(0.0, ns_scale)
ax2.set_xlabel(ns_scale_label)
except ValueError as e:
logger.warning(f"{param}: {e}")
ax.set_xlabel(x_label)
ax.set_ylabel(param)
plt.title("Bias (" + param + ")")
savepath = os.path.join(self.plot_dir,
"bias_" + param + ".pdf")
logger.info("Saving bias plot to {0}".format(savepath))
try:
os.makedirs(os.path.dirname(savepath))
except OSError:
pass
plt.tight_layout()
plt.savefig(savepath)
except KeyError as e:
logger.warning(
f"KeyError for {param}: {e}! Can not make bias plots!")
finally:
plt.close()
def find_disc_potential(self):
ts_path = os.path.join(self.plot_dir, "ts_distributions/0.pdf")
try:
bkg_dict = self.results[scale_shortener(0.0)]
except KeyError:
logger.error("No key equal to '0'")
return
bkg_ts = bkg_dict["TS"]
disc_threshold = plot_background_ts_distribution(bkg_ts,
ts_path,
ts_type=self.ts_type)
self.disc_ts_threshold = disc_threshold
bkg_median = np.median(bkg_ts)
x = sorted(self.results.keys())
y = []
y_25 = []
x = [scale_shortener(i) for i in sorted([float(j) for j in x])]
for scale in x:
ts_array = np.array(self.results[scale]["TS"])
frac = float(len(ts_array[ts_array > disc_threshold])) / (float(
len(ts_array)))
logger.info(
"Fraction of overfluctuations is {0:.2f} above {1:.2f} (N_trials={2}) (Scale={3})"
.format(frac, disc_threshold, len(ts_array), scale))
y.append(frac)
frac_25 = float(len(ts_array[ts_array > 25.0])) / (float(
len(ts_array)))
logger.info(
"Fraction of overfluctuations is {0:.2f} above 25 (N_trials={1}) (Scale={2})"
.format(frac_25, len(ts_array), scale))
y_25.append(frac_25)
self.make_plots(scale)
x = np.array([float(s) for s in x])
x_flux = k_to_flux(x)
threshold = 0.5
sols = []
for i, y_val in enumerate([y, y_25]):
def f(x, a, b, c):
value = scipy.stats.gamma.cdf(x, a, b, c)
return value
best_f = None
try:
res = scipy.optimize.curve_fit(
f, x, y_val, p0=[6, -0.1 * max(x), 0.1 * max(x)])
best_a = res[0][0]
best_b = res[0][1]
best_c = res[0][2]
def best_f(x):
return f(x, best_a, best_b, best_c)
sol = scipy.stats.gamma.ppf(0.5, best_a, best_b, best_c)
setattr(self, ["disc_potential", "disc_potential_25"][i],
k_to_flux(sol))
except RuntimeError as e:
logger.warning(f"RuntimeError for discovery potential!: {e}")
xrange = np.linspace(0.0, 1.1 * max(x), 1000)
savepath = os.path.join(self.plot_dir,
"disc" + ["", "_25"][i] + ".pdf")
fig = plt.figure()
ax1 = fig.add_subplot(111)
ax1.scatter(x_flux, y_val, color="black")
if not isinstance(best_f, type(None)):
ax1.plot(k_to_flux(xrange), best_f(xrange), color="blue")
ax1.axhline(threshold, lw=1, color="red", linestyle="--")
ax1.axvline(self.sensitivity, lw=2, color="black", linestyle="--")
ax1.axvline(self.disc_potential, lw=2, color="red")
ax1.set_ylim(0.0, 1.0)
ax1.set_xlim(0.0, k_to_flux(max(xrange)))
ax1.set_ylabel(
r"Overfluctuations relative to 5 $\sigma$ Threshold")
plt.xlabel(
r"Flux Normalisation @ 1GeV [ GeV$^{-1}$ cm$^{-2}$ s$^{-1}$]")
if not np.isnan(self.flux_to_ns):
ax2 = ax1.twiny()
ax2.grid(0)
ax2.set_xlim(0.0, self.flux_to_ns * k_to_flux(max(xrange)))
ax2.set_xlabel(r"Number of neutrinos")
fig.savefig(savepath)
plt.close()
if self.disc_potential > max(x_flux):
self.extrapolated_disc = True
msg = ""
if self.extrapolated_disc:
msg = "EXTRAPOLATED "
logger.info("{0}Discovery Potential is {1:.3g}".format(
msg, self.disc_potential))
logger.info("Discovery Potential (TS=25) is {0:.3g}".format(
self.disc_potential_25))
def merge_pickle_data(self):
all_sub_dirs = [
x for x in os.listdir(self.pickle_output_dir)
if x[0] != "." and x != "merged"
]
try:
os.makedirs(self.merged_dir)
except OSError:
pass
for sub_dir_name in all_sub_dirs:
sub_dir = os.path.join(self.pickle_output_dir, sub_dir_name)
files = os.listdir(sub_dir)
merged_path = os.path.join(self.merged_dir, sub_dir_name + ".pkl")
if os.path.isfile(merged_path):
logger.debug(f"loading merged data from {merged_path}")
with open(merged_path, "rb") as mp:
merged_data = Pickle.load(mp)
else:
merged_data = {}
for filename in files:
path = os.path.join(sub_dir, filename)
try:
with open(path, "rb") as f:
data = Pickle.load(f)
except (EOFError, IsADirectoryError):
logger.warning("Failed loading: {0}".format(path))
continue
os.remove(path)
if merged_data == {}:
merged_data = data
else:
for (key, info) in data.items():
if isinstance(info, list):
merged_data[key] += info
else:
for (param_name, params) in info.items():
try:
merged_data[key][param_name] += params
except KeyError as m:
logger.warning(
f"Keys [{key}][{param_name}] not found in \n {merged_data}"
)
raise KeyError(m)
with open(merged_path, "wb") as mp:
Pickle.dump(merged_data, mp)
if len(list(merged_data.keys())) > 0:
self.results[scale_shortener(
float(sub_dir_name))] = merged_data
if len(list(self.results.keys())) == 0:
logger.warning("No data was found by ResultsHandler object! \n")
logger.warning("Tried root directory: \n {0} \n ".format(
self.pickle_output_dir))
sys.exit()
def scales(self):
"""directly return the injected scales"""
scales = [scale_shortener(i) for i in self.scales_float]
return scales