def JSON2I3Summary(j):
from icecube.dataclasses import I3MapStringDouble
import json
d = json.loads(j)
summary = I3MapStringDouble()
for key,value in d.items():
summary[str(key)] = value
return summary
def NoiseWeight(frame): weightmap = I3MapStringDouble() weightmap["simulated_livetime"] = 25*I3Units.millisecond * options.NUMEVENTS weightmap["estimated_time_buffer"] = 20*I3Units.microsecond * options.NUMEVENTS weightmap["muon_flux_rate"] = 2800*I3Units.hertz weightmap["muon_trigger_time"] = 30*I3Units.microsecond weightmap["weight"] = 1/( (weightmap["simulated_livetime"]-weightmap["estimated_time_buffer"]) * (1 - weightmap["muon_flux_rate"] * weightmap["muon_trigger_time"]) ) frame["noise_weight"] = weightmap
def DAQ(self, frame):
if frame.Has(self.genie_name):
genie_res = frame[self.genie_name]
genie_info = I3MapStringDouble()
for key in [
"diffxsec", "xsec", "Q2", "Q2s", "W", "Ws", "y", "ys", "x",
"xs", "t", "ts"
]:
if not key in genie_res.keys():
continue
genie_info[key] = genie_res[key]
frame[self.output_name] = genie_info
self.PushFrame(frame)
return True
def write_weight_dict(frame,weight_dict_name) :
# Create a weight dict if doesn't already exist
# Or grab an existing one (removing it from frame ready to write the new one)
if weight_dict_name in frame :
weight_dict = frame[weight_dict_name]
frame.Delete(weight_dict_name)
else :
from icecube.dataclasses import I3MapStringDouble
weight_dict = I3MapStringDouble()
# Write the event generation properties
weight_dict["power_law_index"] = power_law_index
weight_dict["power_law_offset"] = power_law_offset
weight_dict["min_energy"] = min_energy
weight_dict["max_energy"] = max_energy
weight_dict["min_multiplicity"] = model.flux.min_multiplicity
weight_dict["max_multiplicity"] = model.flux.max_multiplicity
weight_dict["outer_surface_center_x"] = outer_surface.center.x
weight_dict["outer_surface_center_y"] = outer_surface.center.y
weight_dict["outer_surface_center_z"] = outer_surface.center.z
weight_dict["outer_surface_length"] = outer_surface.length
weight_dict["outer_surface_radius"] = outer_surface.radius
weight_dict["use_inner_surface"] = int(args.inner_cylinder)
if args.inner_cylinder :
weight_dict["inner_surface_center_x"] = inner_surface.center.x
weight_dict["inner_surface_center_y"] = inner_surface.center.y
weight_dict["inner_surface_center_z"] = inner_surface.center.z
weight_dict["inner_surface_length"] = inner_surface.length
weight_dict["inner_surface_radius"] = inner_surface.radius
weight_dict["use_kde"] = int(args.kde)
# Add weight information (if KDE not run, which handles this)
#TODO Need to better integrate this with the KDE code
if not args.kde :
assert frame.Has(raw_weight_name), "Muon weight '%s' not found in frame" % raw_weight_name
raw_weight = frame[raw_weight_name].value # Weight before normalisation or KDE prescale
num_events = float(args.numevents)
weight = raw_weight / num_events # scale by probability of number of events in file #TODO Isn;t the generator (which is passed to the weighter) already aware of the num events?
weight_dict["raw_weight"] = raw_weight
weight_dict["num_events"] = num_events
weight_dict["weight"] = weight
# Write the weight dict
frame.Put(weight_dict_name,weight_dict)
def DAQ(self, frame):
if frame.Has(self.genie_name):
genie_res = frame[self.genie_name]
MCweight = frame["I3MCWeightDict"]
if "GENIEWeight" in MCweight:
wgt_zero = MCweight["GENIEWeight"]
else:
return True
genie_syst = I3MapStringDouble()
for key in genie_res.keys():
if 'rw' in key:
syst_values = np.array(genie_res[key])
for i in range(0, 4):
genie_syst[key + '_%i' % i] = syst_values[i]
frame[self.output_name] = genie_syst
# somehow there is no '_syststeps' in the keys, so this if statement is useless
if "_syststeps" in genie_res.keys():
print "step 5.1"
genie_syst = I3MapStringVectorDouble()
i_n_sigma = genie_res["_syststeps"]
n_i_sigma = dict()
for i in xrange(0, len(i_n_sigma)):
n_i_sigma[i_n_sigma[i]] = i
sorted_sigma = n_i_sigma.keys()
sorted_sigma.sort()
genie_syst["n_sigma"] = sorted_sigma
for key in genie_res.keys():
if 'rw_' in key:
syst_values = []
orig_syst_values = genie_res[key]
all_syst_1 = True
for ns in sorted_sigma:
new_syst_i = n_i_sigma[ns]
if orig_syst_values[new_syst_i] != 1:
all_syst_1 = False
break
for ns in sorted_sigma:
new_syst_i = n_i_sigma[ns]
if not all_syst_1:
ratio = orig_syst_values[new_syst_i] / wgt_zero
else:
ratio = 1.
syst_values.append(ratio)
genie_syst[key[3:]] = syst_values
frame[self.output_name] = genie_syst
self.PushFrame(frame)
return True
def NoiseWeight(frame, NEvents):
"""
Write a weightmap to the frame for the noise triggers. REQUIRED for the CoincidenceAfterProcessing module. This
should only depend on the number of events, since there's no physics or flux models to worry about to first order.
There is also a "time buffer", which is meant to take into account edge effects of triggers near the frame edges.
:param NumEvents: Number of events being produced for this file
"""
from icecube.dataclasses import I3MapStringDouble, I3MCTree
from icecube.icetray import I3Units
weightmap = I3MapStringDouble()
weightmap["simulated_livetime"] = 100 * I3Units.millisecond
weightmap["time_buffer"] = 2 * 20 * I3Units.microsecond
weightmap["nevents"] = NEvents
weightmap["weight"] = 1.0 / (
weightmap["nevents"] *
(weightmap["simulated_livetime"] - weightmap["time_buffer"]))
frame["noise_weight"] = weightmap
return
def oscNext_L7_compute_final_reconstructed_values(frame, cleaned_pulses):
'''
Comoute the final reconsturcted values we will use for analysis.
We use RetroReco as the final level reconstruction, and apply a number of improved conversion
factors and correction factors to get good agreement with the truth parameters.
Also do a bunch of other post-processing such as:
- Computing reduced LLH
- Extract uncertainties on reconstructed quantities
- Compute derived quantities like rho36
'''
from icecube.dataclasses import I3Double, I3MapStringDouble
from icecube.oscNext.frame_objects.geom import calc_rho_36
from icecube.oscNext.frame_objects.reco import convert_retro_reco_energy_to_neutrino_energy
from retro.i3processing.retro_recos_to_i3files import const_en2len, const_en_to_gms_en, GMS_LEN2EN
#
# Compute reco rho36
#
retro_rho36 = I3MapStringDouble()
for key in ['mean', 'median', 'upper_bound', 'lower_bound', 'max']:
x = frame['retro_crs_prefit__x'][key]
y = frame['retro_crs_prefit__y'][key]
z = frame['retro_crs_prefit__z'][key]
r36 = calc_rho_36(x=x, y=y)
retro_rho36[key] = r36
frame[L7_RETRO_RECO_RHO36_KEY] = retro_rho36
#
# Track length/energy
#
#
# To get the length, we have to reverse-compute it from the
# level6 track_energy estimate that is already available in the frame
# This energy was computed using a constant light-emission approximation
# so the length is simply the energy of the track divided by a constant
# factor of 0.22 GeV/m
#
track_energy_const = frame["retro_crs_prefit__track_energy"]["median"]
track_length = const_en2len(track_energy_const)
#
# Overall energy
#
# The cascade energy fitted by retro reco assumed an EM cascade
frame['L7_reconstructed_em_cascade_energy'] = I3Double(
frame["retro_crs_prefit__cascade_energy"]["median"])
# Some additional steps are required to get the final energy values from the RetroReco fits
# 1) Convert from EM energy to hadronic-equivalent energy
# 2) Apply some scaling factors to the hadronic cascade energy and the track length to better match the truth params on average
# 3) Compute track energy again using the corrected length, and also now using the GMS tables which work better than the constant energy loss approximatiin
# 4) Sum the track and cascade energy to get the total energy
cascade_energy, track_energy, total_energy, track_length = convert_retro_reco_energy_to_neutrino_energy(
em_cascade_energy=frame['L7_reconstructed_em_cascade_energy'].value,
track_length=track_length,
)
# Write final variable to frame
frame[L7_FINAL_RECO_TRACK_LENGTH_KEY] = I3Double(track_length)
frame[L7_FINAL_RECO_CASCADE_ENERGY_KEY] = I3Double(cascade_energy)
frame[L7_FINAL_RECO_TRACK_ENERGY_KEY] = I3Double(track_energy)
frame[L7_FINAL_RECO_TOTAL_ENERGY_KEY] = I3Double(total_energy)
#
# Store the other reconstructed params
#
# Directly using the retro reco value for all other params
frame[L7_FINAL_RECO_X_KEY] = I3Double(
frame['retro_crs_prefit__x']['median'])
frame[L7_FINAL_RECO_Y_KEY] = I3Double(
frame['retro_crs_prefit__y']['median'])
frame[L7_FINAL_RECO_Z_KEY] = I3Double(
frame['retro_crs_prefit__z']['median'])
frame[L7_FINAL_RECO_RHO36_KEY] = I3Double(
frame[L7_RETRO_RECO_RHO36_KEY]['median'])
frame[L7_FINAL_RECO_ZENITH_KEY] = I3Double(
frame['retro_crs_prefit__zenith']['median'])
frame[L7_FINAL_RECO_AZIMUTH_KEY] = I3Double(
frame['retro_crs_prefit__azimuth']['median'])
frame[L7_FINAL_RECO_TIME_KEY] = I3Double(
frame['retro_crs_prefit__time']['median'])
#
# Compute the llh-based error
#
# Get the uncertainties on the RetroReco reconstructed quantities
for qty in [
'azimuth', 'zenith', 'x', 'y', 'z', 'cascade_energy',
'track_energy', 'energy', 'time', 'zero_dllh'
]:
ub = frame['retro_crs_prefit__%s' % (qty)]['upper_bound']
lb = frame['retro_crs_prefit__%s' % (qty)]['lower_bound']
md = frame['retro_crs_prefit__%s' % (qty)]['median']
p1s = ub - md # Positive 1 sigma
n1s = md - lb # Negative 1 sigma
w1s = p1s + n1s # Full 1 sigma width (-1sigma -> +1sigma)
frame['L7_retro_crs_prefit__%s_sigma_p' % (qty)] = I3Double(p1s)
frame['L7_retro_crs_prefit__%s_sigma_n' % (qty)] = I3Double(n1s)
frame['L7_retro_crs_prefit__%s_sigma_tot' % (qty)] = I3Double(w1s)
#
# Compute the Reduced Goodness-of-fit
#
# Want to reduce compute the redued LLH value from RetroReco
# This mans dividng it by the umber of elements used in the reco, e.g. num DOMs
# Need to re-produce the pulse pre-cleaning RetroReco does in order to get the right value
if frame.Has(cleaned_pulses):
try:
pulse_serie = frame[cleaned_pulses].apply(frame)
except:
pulse_serie = frame[cleaned_pulses]
nch = 0
nch_retro = 0
nstring_triggered = []
for dom in pulse_serie.keys():
# just making sure we don't catch stuff from iceTop...
if dom.om <= 60:
nch += len(pulse_serie[dom]) > 0
nstring_triggered.append(dom.string)
#
# Apply the precleaning that Retro Reco uses
#
# Quantize the charge in bins of 0.05
quanta = 0.05
quantized_charge = np.array([
(np.float64(x.charge) // quanta) * quanta + quanta / 2.
for x in pulse_serie[dom]
])
nch_retro += sum(
quantized_charge >= 0.3
) != 0 # only add the channel if there is 1 pulse above 0.3 pe or more
GOF = frame['retro_crs_prefit__max_postproc_llh'].value
nstrings = float(len(np.unique(nstring_triggered)))
frame['L7_max_postproc_llh_over_nstring'] = I3Double(GOF / nstrings)
frame['L7_max_postproc_llh_over_nch'] = I3Double(GOF / float(nch))
frame['L7_nchannel_used_in_retro'] = I3Double(float(nch_retro))
if nch_retro <= 0:
frame['L7_max_postproc_llh_over_nch_retro'] = I3Double(np.NaN)
else:
frame['L7_max_postproc_llh_over_nch_retro'] = I3Double(
GOF / float(nch_retro))
else:
frame['L7_max_postproc_llh_over_nstring'] = I3Double(np.NaN)
frame['L7_max_postproc_llh_over_nch'] = I3Double(np.NaN)
frame['L7_nchannel_used_in_retro'] = I3Double(np.NaN)
frame['L7_max_postproc_llh_over_nch_retro'] = I3Double(np.NaN)
def BatchProcessBuffer(self, frames):
"""Batch Process a list of frames.
This includes pre-processing, prediction and storage of the results
"""
for counter, pulse_key in enumerate(self.__pulsemap):
f_slices = []
timer_t0 = time.time()
benchmark_times = [timer_t0]
if self.__num_pframes == 0:
return
for frame in frames:
if frame.Stop != icetray.I3Frame.Physics:
continue
if pulse_key not in frame.keys():
print('No Pulsemap called {}..continue without prediction'.format(pulse_key))
continue
f_slice = []
pulses = self.get_cleaned_pulses(
frame, pulse_key,
bright_dom_key = self.__bright_doms_key[counter],
bad_dom_key = self.__bad_dom_key[counter],
calib_err_key = self.__calib_err_key[counter],
sat_window_key = self.__sat_window_key[counter])
t0 = get_t0(pulses)
for key in self.__inp_shapes.keys():
f_slice.append(np.zeros(self.__inp_shapes[key]['general']))
for omkey in pulses.keys():
dom = (omkey.string, omkey.om)
if dom not in self.__grid.keys():
continue
if (omkey.string in ic_79_strings) & (self.__is_ic79):
continue
gpos = self.__grid[dom]
charges = np.array([p.charge for p in pulses[omkey][:]
if p.charge > 0])
times = np.array([p.time for p in pulses[omkey][:]
if p.charge > 0]) - t0
widths = np.array([p.width for p in pulses[omkey][:]
if p.charge > 0])
if len(charges) == 0: # might happen through pulse cleaning
continue
for branch_c, inp_branch in enumerate(self.__inputs):
for inp_c, inp in enumerate(inp_branch):
f_slice[branch_c][gpos[0]][gpos[1]][gpos[2]][inp_c] =\
inp[1](eval(inp[0]))
f_slices.append(f_slice)
benchmark_times.append(time.time())
tf.compat.v1.keras.backend.set_session(self.sess)
try:
predictions = self.__model.predict(np.array(np.squeeze(f_slices,
axis=1),
ndmin=5),
batch_size=self.__batch_size,
verbose=0, steps=None)
except Exception as e:
print(e.message, e.args)
return
prediction_time = (time.time() - benchmark_times[-1])/len(f_slices)
benchmark_times = np.diff(benchmark_times)
i = 0
for frame in frames:
if frame.Stop != icetray.I3Frame.Physics:
continue
if pulse_key not in frame.keys():
continue
output = I3MapStringDouble()
prediction = np.concatenate(np.atleast_2d(predictions[i]))
for j in range(len(prediction)):
output[self.__output_names[j]] = float(prediction[j])
frame.Put(self.__save_as[counter], output)
if self.__benchmark & (counter==0):
output_bm = I3MapStringDouble()
output_bm['processing'] = benchmark_times[i]
output_bm['avg_prediction'] = prediction_time
output_bm['batch_size'] = len(f_slices)
frame.Put(self.__save_as[counter] + '_Benchmark', output_bm)
if self.__add_truth and not ('classification_truth' in frame.keys()):
try:
classify_wrapper(frame)
except Exception as e:
print('Failed while calculating the truth...')
print(e)
i += 1
tot_time = time.time() - timer_t0
t_str = 'Total Time {:.2f}s, Processing Time: {:.2f}s/event, Prediction Time {:.3f}s/event \n'
print(t_str.format(tot_time, np.median(benchmark_times), prediction_time))
def extract_all_reco_info(reco, reco_name, reco_suffix):
"""Populate ALL Retro reco information to simple I3-typed fields in the
frame.
Parameters
----------
reco
reco_name : str
reco_suffix : str
Returns
-------
all_reco_info : dict
"""
all_reco_info = OrderedDict()
for field in reco.dtype.names:
key = "{}{}__{}".format(reco_name, reco_suffix, field)
val = reco[field]
if hasattr(val, "dtype") and len(val.dtype) > 0:
# TODO: handle I3MapStringBool, I3MapStringInt?
val = I3MapStringDouble(list(zip(val.dtype.names, val.tolist())))
else:
val_type = getattr(val, "dtype", type(val))
# floating types
if val_type in (
float,
np.float,
np.float_,
np.float16,
np.float32,
np.float64,
):
i3type = I3Double
pytype = float
# integer types
elif val_type in (
int,
np.int,
np.int_,
np.intp,
np.integer,
np.int0,
np.int8,
np.int16,
np.int32,
np.int64,
np.uint,
np.uintp,
np.uint8,
np.uint16,
np.uint32,
np.uint64,
):
i3type = I3Int
pytype = int
# boolean types
elif val_type in (
bool,
np.bool,
np.bool_,
np.bool8,
):
i3type = I3Bool
pytype = bool
else:
raise TypeError(
"Don't know how to handle type {}".format(val_type))
val = i3type(pytype(val))
all_reco_info[key] = val
return all_reco_info