def Physics(self, frame):
"""
Either Generate PDFs or Calc LLHR.
"""
if not self.Use_Laputop:
En = np.log10(frame[self.EnergyRecoName].energy)
else:
En = frame[self.LaputopParamsName].value(
recclasses.LaputopParameter.Log10_S125)
ze = np.cos(frame[self.AngularRecoName].dir.zenith)
llhr = frame[self.ITLLHRName]['llh_ratio']
h, edges = np.histogramdd(sample=np.array([[En], [ze], [llhr]]).T,
bins=self.binedges)
if np.shape(h) != np.shape(self.hist):
log_fatal(
'initialized histogram and fill histogram dont match in shape')
self.hist += h
self.n_events += 1
self.PushFrame(frame)
return
def _create_in_array(self,frame):
if self.EnergyRecoName:
En = np.log10(frame[self.EnergyRecoName].energy)
elif self.LaputopParamsName:
En = np.log10(frame[self.LaputopParamsName].s125)
else:
log_fatal('One of EnergyRecoName_I3Particle or LaputopParamsName needs to be given')
ze = np.cos(frame[self.AngularRecoName].dir.zenith)
hits = frame[self.HitsName]
unhits = frame[self.UnhitsName]
excluded = frame[self.ExcludedName]
#hits_t, hits_q, hits_r = np.array([[signed_log(hit.time_residual),np.log10(hit.charge), log_plus_one(hit.distance)] for hit in hits]).T
hits_t = signed_log(np.array([hit.time_residual for hit in hits]))
hits_q = np.log10(np.array([hit.charge for hit in hits]))
hits_r = log_plus_one(np.array([hit.distance for hit in hits]))
hits_E = np.ones_like(hits_r)*En
hits_z = np.ones_like(hits_r)*ze
#unhits_t, unhits_q, unhits_r = np.array([[signed_log(hit.time_residual),np.log10(hit.charge), log_plus_one(hit.distance)] for hit in unhits]).T
unhits_t = signed_log(np.array([hit.time_residual for hit in unhits]))
unhits_q = np.log10(np.array([hit.charge for hit in unhits]))
unhits_r = log_plus_one(np.array([hit.distance for hit in unhits]))
unhits_E = np.ones_like(unhits_r)*En
unhits_z = np.ones_like(unhits_r)*ze
#excluded_t, excluded_q, excluded_r = np.array([[signed_log(hit.time_residual),np.log10(hit.charge), log_plus_one(hit.distance)] for hit in excluded]).T
excluded_t = signed_log(np.array([hit.time_residual for hit in excluded]))
excluded_q = np.log10(np.array([hit.charge for hit in excluded]))
excluded_r = log_plus_one(np.array([hit.distance for hit in excluded]))
excluded_E = np.ones_like(excluded_r)*En
excluded_z = np.ones_like(excluded_r)*ze
# ready data for entry to 5D hist
t = np.concatenate( (hits_t, unhits_t, excluded_t) )
q = np.concatenate( (hits_q, unhits_q, excluded_q) )
r = np.concatenate( (hits_r, unhits_r, excluded_r) )
E = np.concatenate( (hits_E, unhits_E, excluded_E) )
z = np.concatenate( (hits_z, unhits_z, excluded_z) )
if len(t)!=162 or len(q)!=162 or len(r)!=162:
print 'N_t %s N_q %s N_r %s'%(len(t),len(q),len(r))
log_fatal('Total Tanks in Event not 162')
if np.isnan(t).any() or np.isnan(q).any() or np.isnan(r).any():
print 't',t
print 'q',q
print 'r',r
log_warn('signed_time/logq/logr have nans')
in_array=np.vstack([E,z,q,t,r]).T
return in_array
def Configure(self):
"""
Configure
Load Input Parameters as class members for easy access
"""
self.HitsName = self.GetParameter('Hits_I3VectorShieldHitRecord')
self.UnhitsName = self.GetParameter('Unhits_I3VectorShieldHitRecord')
self.ExcludedName = self.GetParameter(
'Excluded_I3VectorShieldHitRecord')
self.AngularRecoName = self.GetParameter('AngularReco_I3Particle')
self.EnergyRecoName = self.GetParameter('EnergyReco_I3Particle')
self.Use_Laputop = self.GetParameter('Use_Laputop')
self.LaputopParamsName = self.GetParameter('LaputopParamsName')
self.RunMode = self.GetParameter('RunMode')
self.Decimals = self.GetParameter('DecimalsForSanityCheck')
if self.RunMode == 'GeneratePDF':
self.OutputName = self.GetParameter('OutputFileName')
self.binedges = self.GetParameter('BinEdges5D')
self.distinct_regions_binedges = self.GetParameter(
'DistinctRegionsBinEdges3D')
# make sure distinct regions binedges make sense
if len(self.distinct_regions_binedges) == 0:
# give the whole region as a distinct single region
self.distinct_regions_binedges = [self.binedges[2:]]
else:
#check that each distinct region binedge is same shape as self.binedges i.e.
for i in self.distinct_regions_binedges:
if np.shape(i) != np.shape(self.binedges[2:]):
print 'shape of self.binedges[2:] :', np.shape(
self.binedges[2:])
print 'shape of self.distinct_regions_binedges', np.shape(
self.distinct_regions_binedges)
log_fatal(
'DistinctRegionBinEdges and BinEdges* not compatible'
)
#check that joining all distinct regions gives total binedges
check_distinct_regions_add_up_to_full(
self.distinct_regions_binedges,
self.binedges[2:],
decimals=self.Decimals)
self.labels = ['logE', 'cosZ', 'logQ', 'signedlogT', 'logRplusOne']
#creates the self.hist
self._init_hist()
elif self.RunMode == 'CalcLLHR':
self.SigPDFInputName = self.GetParameter('SigPDFInputFileName')
self.BkgPDFInputName = self.GetParameter('BkgPDFInputFileName')
# this one should create self.bkg_hist, self.sig_hist, self.binedges, self.labels, self.distinct_regions_binedges
self._load_PDF_from_file()
self.SubtractEventFromPDF = self.GetParameter(
'SubtractEventFromPDF')
self.objname = self.GetParameter('Output')
return
def _fill(self,sample):
h,edges=np.histogramdd(sample,self.binedges)
if np.shape(h)!=np.shape(self.hist):
log_fatal('initialized histogram and fill histogram dont match in shape')
self.hist+= h
self.n_events+=1
return
def Physics(self,frame):
if self.RunMode=='GeneratePDF':
self._GenPDFsPhysics(frame)
elif self.RunMode=='CalcLLHR':
self._CalcLLHRPhysics(frame)
else:
log_fatal('RunMode can only accept one these two inputs: GeneratePDF / CalcLLHR')
self.PushFrame(frame)
return
def __init__(self, encoder, inpdir, outfile):
if encoder == 'ffmpeg':
cmd = ['ffmpeg', '-i', '{0}/frame%08d.png'.format(inpdir)]
cmd += ffmpeg_flags
elif encoder == 'avconv':
cmd = ['avconv', '-i', '{0}/frame%08d.png'.format(inpdir)]
cmd += avconv_flags
elif encoder == 'mencoder':
cmd = ['mencoder', 'mf://{0}/*.png'.format(inpdir)]
cmd += mencoder_flags
else:
i3logging.log_fatal("bad encoder argument: {0}".format(encoder))
cmd += [outfile]
i3logging.log_debug('Encoding cmdline: ' + ' '.join(cmd))
self.proc = subprocess.Popen(cmd)
def _fill(self, sample):
"""
Pretty much all that is done during GeneratePDF mode.
i.e. to fill up the histogram initialized during Configure.
"""
h, edges = np.histogramdd(sample=sample, bins=self.binedges)
if np.shape(h) != np.shape(self.hist):
log_fatal(
'initialized histogram and fill histogram dont match in shape')
self.hist += h
self.n_events += 1
#print self.n_events, np.sum(self.hist), np.sum(self.hist)/162., np.sum(h)
#print RAM_test()
return
def calc_LLHR(in_array, sig_hist, bkg_hist, binedges,
distinct_regions_binedges, SubtractEventFromPDF):
from icecube.icetray.i3logging import log_fatal, log_warn
from llh_ratio_nd import get_slice_vector, log_likelihood_ratio
d = {}
d['llh_ratio'] = 0.
d['n_extrapolations_sig_PDF'] = 0.
d['n_extrapolations_bkg_PDF'] = 0.
d['llh_sig'] = 0.
d['llh_bkg'] = 0.
d['isGood'] = 0.
d['tanks_have_nans'] = 0.
logE = in_array[0][0]
coszen = in_array[0][1]
# check if event logE and coszen lies within range of binedges
if logE > binedges[0][-1] or logE < binedges[0][0]:
return d
if coszen > binedges[1][-1] or coszen < binedges[1][0]:
return d
# find the logE and coszen bins select those bins in sig/bkg pdfs
logEbincenters = np.array((binedges[0][1:] + binedges[0][:-1]) / 2.)
coszenbincenters = np.array((binedges[1][1:] + binedges[1][:-1]) / 2.)
dE = np.absolute(logEbincenters - logE)
Ebin = np.where(np.amin(dE) == dE)[0][0]
dcZ = np.absolute(coszenbincenters - coszen)
cZbin = np.where(np.amin(dcZ) == dcZ)[0][0]
sig_hist = sig_hist[Ebin][cZbin]
bkg_hist = bkg_hist[Ebin][cZbin]
# select Q, T, R dimensions, generate event histogram
in_array = (in_array.T[2:]).T
binedges = binedges[2:]
event_hist, temp = np.histogramdd(in_array, binedges)
# store status if any of the q, t, r values have nans in this event
if np.isnan(in_array[0][0]).any() or np.isnan(
in_array[0][1]).any() or np.isnan(in_array[0][2]).any():
d['tanks_have_nans'] = 1.
# subtract the event from the PDF if it was used for generating the PDF
if SubtractEventFromPDF:
if SubtractEventFromPDF == 'Sig':
sig_hist = sig_hist - event_hist
if (sig_hist < 0).any():
log_fatal('Event subtraction led to negative values')
if SubtractEventFromPDF == 'Bkg':
bkg_hist = bkg_hist - event_hist
if (bkg_hist < 0).any():
log_fatal('Event subtraction led to negative values')
# normalize histogram, obtain PDFs
sig_pdf = sig_hist / np.sum(sig_hist)
bkg_pdf = bkg_hist / np.sum(bkg_hist)
# calculate llh ratio for each region separately and add it up
# separate calculation is done to avoid one region influencing
# extrapolated values of empty pixels in the PDF in another region
llh_map_sig = np.zeros_like(sig_hist)
llh_map_bkg = np.zeros_like(bkg_hist)
d['isGood'] = 1.
for region_edges in distinct_regions_binedges:
# obtain slice vector for the region of the PDF
region_range = [[i[0], i[-1]] for i in region_edges]
slice_vector = get_slice_vector(binedges, region_range)
temp = log_likelihood_ratio(heatmap1=sig_pdf[slice_vector],
heatmap2=bkg_pdf[slice_vector],
event_hist=event_hist[slice_vector])
d['llh_ratio'] += temp[0]
# all the rest are debugging variables. some will be stored in I3VectorMap.
# not storing any histograms as output. Just numbers.
d['n_extrapolations_sig_PDF'] += temp[1]
d['n_extrapolations_bkg_PDF'] += temp[2]
d['llh_sig'] += temp[5]
d['llh_bkg'] += temp[6]
if np.isnan(temp[5]) or np.isnan(temp[6]):
d['isGood'] = 0.
# the following diagnostics are not being returned
extrapolated_sig_PDF = temp[3]
extrapolated_bkg_PDF = temp[4]
llh_map_sig[slice_vector] = temp[7]
llh_map_bkg[slice_vector] = temp[8]
return d
def Physics(self, frame):
#initiate output vectors
hits = recclasses.I3VectorShieldHitRecord()
unhits = recclasses.I3VectorShieldHitRecord()
excluded = recclasses.I3VectorShieldHitRecord()
not_unhit = [] # will keep track of hit+excluded doms
#load reconstruction
axis = frame[self.reco_name]
rotation = to_shower_cs(axis)
origin = np.array([[axis.pos.x], [axis.pos.y], [axis.pos.z]])
# populate hits vector
for tankpulses in [self.slc_name, self.hlc_name]:
pulses = dataclasses.I3RecoPulseSeriesMap.from_frame(
frame, tankpulses)
for omkey, pulse_items in pulses.iteritems():
if not omkey in self.geometry.omgeo:
log_fatal("OM {om} not in geometry!".format(om=k))
if omkey[1] > 64 or omkey[1] < 61:
log_fatal("OM {om} not an IceTop DOM!".format(om=k))
if len(pulse_items) != 1:
log_fatal(
'only one pulse per omkey should be present. something might be fishy here.'
)
not_unhit.append(omkey)
for pulse in pulse_items:
#calculate the tank's location/ time in shower frame ref
# time is w.r.t. planar shower front arrival time at tank location
position = self.geometry.omgeo[omkey].position
det_cs_position = np.array([[position.x], [position.y],
[position.z]])
shower_cs_position = rotation * (det_cs_position - origin)
shower_cs_radius = np.sqrt(shower_cs_position[0]**2 +
shower_cs_position[1]**2)
time = pulse.time - float(
axis.time - shower_cs_position[2] / I3Constants.c)
#store all the pulse information in I3ShieldHitRecord and add it to hits vector
new_pulse = recclasses.I3ShieldHitRecord()
new_pulse.DOMkey = omkey
new_pulse.charge = pulse.charge
new_pulse.time_residual = time
new_pulse.distance = np.float(shower_cs_radius)
hits.append(new_pulse)
# populate excluded vector
tanklist = frame[self.excluded_name]
for tank in tanklist:
not_unhit.append(tank.default_omkey)
new_pulse = recclasses.I3ShieldHitRecord()
# new_pulse.DOMkey = tank
new_pulse.DOMkey = tank.default_omkey
new_pulse.charge = self.exc_fake_q
new_pulse.time_residual = self.exc_fake_t
position = self.geometry.omgeo[new_pulse.DOMkey].position
det_cs_position = np.array([[position.x], [position.y],
[position.z]])
shower_cs_position = rotation * (det_cs_position - origin)
shower_cs_radius = np.sqrt(shower_cs_position[0]**2 +
shower_cs_position[1]**2)
new_pulse.distance = np.float(shower_cs_radius)
excluded.append(new_pulse)
#populate unhits vector
stations = frame['I3Geometry'].stationgeo
for station in stations:
for tank in station.data():
is_unhit = 0
for dom in tank.omkey_list:
if dom not in not_unhit:
is_unhit += 1
if is_unhit == 2:
new_pulse = recclasses.I3ShieldHitRecord()
new_pulse.DOMkey = dom # pick the last one in the loop above
new_pulse.charge = self.unhit_fake_q
new_pulse.time_residual = self.unhit_fake_t
position = self.geometry.omgeo[new_pulse.DOMkey].position
det_cs_position = np.array([[position.x], [position.y],
[position.z]])
shower_cs_position = rotation * (det_cs_position - origin)
shower_cs_radius = np.sqrt(shower_cs_position[0]**2 +
shower_cs_position[1]**2)
new_pulse.distance = np.float(shower_cs_radius)
unhits.append(new_pulse)
#put the results to frame
frame.Put(self.HitsName, hits)
frame.Put(self.UnhitsName, unhits)
frame.Put(self.ExcludedName, excluded)
self.PushFrame(frame)
return
def _CalcLLHRPhysics(self, frame):
d = {}
d['llh_ratio'] = 0.
d['n_extrapolations_sig_PDF'] = 0.
d['n_extrapolations_bkg_PDF'] = 0.
d['llh_sig'] = 0.
d['llh_bkg'] = 0.
d['isGood'] = 0.
# load event information
in_array = self._create_in_array(frame)
logE = in_array[0][0]
coszen = in_array[0][1]
# select Q, T, R dimensions, generate event histogram
in_array = (in_array.T[2:]).T
binedges = self.binedges[2:]
event_hist, temp = np.histogramdd(in_array, binedges)
# check if event logE and coszen lies within range of binedges
if logE > self.binedges[0][-1] or logE < self.binedges[0][0]:
frame.Put(self.objname, dataclasses.I3MapStringDouble(d))
return
if coszen > self.binedges[1][-1] or coszen < self.binedges[1][0]:
frame.Put(self.objname, dataclasses.I3MapStringDouble(d))
return
# find the logE and coszen bins select those bins in sig/bkg pdfs
logEbincenters = np.array(
(self.binedges[0][1:] + self.binedges[0][:-1]) / 2.)
coszenbincenters = np.array(
(self.binedges[1][1:] + self.binedges[1][:-1]) / 2.)
dE = np.absolute(logEbincenters - logE)
Ebin = np.where(np.amin(dE) == dE)[0][0]
dcZ = np.absolute(coszenbincenters - coszen)
cZbin = np.where(np.amin(dcZ) == dcZ)[0][0]
sig_hist = self.sig_hist[Ebin][cZbin]
bkg_hist = self.bkg_hist[Ebin][cZbin]
# subtract the event from the PDF if it was used for generating the PDF
if self.SubtractEventFromPDF:
if self.SubtractEventFromPDF == 'Sig':
sig_hist = sig_hist - event_hist
if (sig_hist < 0).any():
log_fatal('Event subtraction led to negative values')
if self.SubtractEventFromPDF == 'Bkg':
bkg_hist = bkg_hist - event_hist
if (bkg_hist < 0).any():
log_fatal('Event subtraction led to negative values')
# normalize histogram, obtain PDFs
sig_pdf = sig_hist / np.sum(sig_hist)
bkg_pdf = bkg_hist / np.sum(bkg_hist)
# calculate llh ratio for each region separately and add it up
# separate calculation is done to avoid one region influencing
# extrapolated values of empty pixels in the PDF in another region
llh_map_sig = np.zeros_like(sig_hist)
llh_map_bkg = np.zeros_like(bkg_hist)
d['isGood'] = 1.
for region_edges in self.distinct_regions_binedges:
# obtain slice vector for the region of the PDF
region_range = [[i[0], i[-1]] for i in region_edges]
slice_vector = get_slice_vector(binedges, region_range)
temp = log_likelihood_ratio(heatmap1=sig_pdf[slice_vector],
heatmap2=bkg_pdf[slice_vector],
event_hist=event_hist[slice_vector])
d['llh_ratio'] += temp[0]
# all the rest are debugging variables. some will be stored in I3VectorMap.
# not storing any histograms as output. Just numbers.
d['n_extrapolations_sig_PDF'] += temp[1]
d['n_extrapolations_bkg_PDF'] += temp[2]
d['llh_sig'] += temp[5]
d['llh_bkg'] += temp[6]
extrapolated_sig_PDF = temp[3]
extrapolated_bkg_PDF = temp[4]
llh_map_sig[slice_vector] = temp[7]
llh_map_bkg[slice_vector] = temp[8]
frame.Put(self.objname, dataclasses.I3MapStringDouble(d))
return
def _create_in_array(self, frame, time_transformation=signed_log):
"""
Create the IceTop specific input array that goes into
GeneratePDF / CalcLLHR .
This is method will need to be
adapted for each analysis.
"""
if not self.Use_Laputop:
En = np.log10(frame[self.EnergyRecoName].energy)
else:
En = frame[self.LaputopParamsName].value(
recclasses.LaputopParameter.Log10_S125)
ze = np.cos(frame[self.AngularRecoName].dir.zenith)
hits = frame[self.HitsName]
unhits = frame[self.UnhitsName]
excluded = frame[self.ExcludedName]
hits_t = time_transformation(
np.array([hit.time_residual for hit in hits]))
hits_q = np.log10(np.array([hit.charge for hit in hits]))
hits_qunlog = np.array([hit.charge for hit in hits])
hits_r = log_plus_one(np.array([hit.distance for hit in hits]))
hits_E = np.ones_like(hits_r) * En
hits_z = np.ones_like(hits_r) * ze
hits_doms = np.array([hit.DOMkey for hit in hits])
if np.isnan(hits_q).any():
select = np.isnan(hits_q)
log_error('nan q doms in %s' % self.HitsName)
log_error(
zip(hits_q[select], hits_qunlog[select], hits_doms[select]))
if np.isnan(hits_t).any():
select = np.isnan(hits_t)
log_error('nan t doms in %s' % self.HitsName)
log_error(zip(hits_t[select], hits_doms[select]))
unhits_t = time_transformation(
np.array([hit.time_residual for hit in unhits]))
unhits_q = np.log10(np.array([hit.charge for hit in unhits]))
unhits_r = log_plus_one(np.array([hit.distance for hit in unhits]))
unhits_E = np.ones_like(unhits_r) * En
unhits_z = np.ones_like(unhits_r) * ze
excluded_t = time_transformation(
np.array([hit.time_residual for hit in excluded]))
excluded_q = np.log10(np.array([hit.charge for hit in excluded]))
excluded_r = log_plus_one(np.array([hit.distance for hit in excluded]))
excluded_E = np.ones_like(excluded_r) * En
excluded_z = np.ones_like(excluded_r) * ze
# ready data for entry to 5D hist
t = np.concatenate((hits_t, unhits_t, excluded_t))
q = np.concatenate((hits_q, unhits_q, excluded_q))
r = np.concatenate((hits_r, unhits_r, excluded_r))
E = np.concatenate((hits_E, unhits_E, excluded_E))
z = np.concatenate((hits_z, unhits_z, excluded_z))
if len(t) != 162 or len(q) != 162 or len(r) != 162:
log_error('N_t %s N_q %s N_r %s' % (len(t), len(q), len(r)))
log_fatal('Total Tanks in Event not 162')
if np.isnan(t).any() or np.isnan(q).any() or np.isnan(r).any():
#print 't',t
#print 'q',q
#print 'r',r
log_warn(
'signed_time/logq/logr have nans, logs125%.2f coszen%.2f' %
(En, ze))
in_array = np.vstack([E, z, q, t, r]).T
return in_array
