def Physics(self, frame): """ Either Generate PDFs or Calc LLHR. """ if not self.Use_Laputop: En = np.log10(frame[self.EnergyRecoName].energy) else: En = frame[self.LaputopParamsName].value( recclasses.LaputopParameter.Log10_S125) ze = np.cos(frame[self.AngularRecoName].dir.zenith) llhr = frame[self.ITLLHRName]['llh_ratio'] h, edges = np.histogramdd(sample=np.array([[En], [ze], [llhr]]).T, bins=self.binedges) if np.shape(h) != np.shape(self.hist): log_fatal( 'initialized histogram and fill histogram dont match in shape') self.hist += h self.n_events += 1 self.PushFrame(frame) return
def _create_in_array(self,frame): if self.EnergyRecoName: En = np.log10(frame[self.EnergyRecoName].energy) elif self.LaputopParamsName: En = np.log10(frame[self.LaputopParamsName].s125) else: log_fatal('One of EnergyRecoName_I3Particle or LaputopParamsName needs to be given') ze = np.cos(frame[self.AngularRecoName].dir.zenith) hits = frame[self.HitsName] unhits = frame[self.UnhitsName] excluded = frame[self.ExcludedName] #hits_t, hits_q, hits_r = np.array([[signed_log(hit.time_residual),np.log10(hit.charge), log_plus_one(hit.distance)] for hit in hits]).T hits_t = signed_log(np.array([hit.time_residual for hit in hits])) hits_q = np.log10(np.array([hit.charge for hit in hits])) hits_r = log_plus_one(np.array([hit.distance for hit in hits])) hits_E = np.ones_like(hits_r)*En hits_z = np.ones_like(hits_r)*ze #unhits_t, unhits_q, unhits_r = np.array([[signed_log(hit.time_residual),np.log10(hit.charge), log_plus_one(hit.distance)] for hit in unhits]).T unhits_t = signed_log(np.array([hit.time_residual for hit in unhits])) unhits_q = np.log10(np.array([hit.charge for hit in unhits])) unhits_r = log_plus_one(np.array([hit.distance for hit in unhits])) unhits_E = np.ones_like(unhits_r)*En unhits_z = np.ones_like(unhits_r)*ze #excluded_t, excluded_q, excluded_r = np.array([[signed_log(hit.time_residual),np.log10(hit.charge), log_plus_one(hit.distance)] for hit in excluded]).T excluded_t = signed_log(np.array([hit.time_residual for hit in excluded])) excluded_q = np.log10(np.array([hit.charge for hit in excluded])) excluded_r = log_plus_one(np.array([hit.distance for hit in excluded])) excluded_E = np.ones_like(excluded_r)*En excluded_z = np.ones_like(excluded_r)*ze # ready data for entry to 5D hist t = np.concatenate( (hits_t, unhits_t, excluded_t) ) q = np.concatenate( (hits_q, unhits_q, excluded_q) ) r = np.concatenate( (hits_r, unhits_r, excluded_r) ) E = np.concatenate( (hits_E, unhits_E, excluded_E) ) z = np.concatenate( (hits_z, unhits_z, excluded_z) ) if len(t)!=162 or len(q)!=162 or len(r)!=162: print 'N_t %s N_q %s N_r %s'%(len(t),len(q),len(r)) log_fatal('Total Tanks in Event not 162') if np.isnan(t).any() or np.isnan(q).any() or np.isnan(r).any(): print 't',t print 'q',q print 'r',r log_warn('signed_time/logq/logr have nans') in_array=np.vstack([E,z,q,t,r]).T return in_array
def Configure(self): """ Configure Load Input Parameters as class members for easy access """ self.HitsName = self.GetParameter('Hits_I3VectorShieldHitRecord') self.UnhitsName = self.GetParameter('Unhits_I3VectorShieldHitRecord') self.ExcludedName = self.GetParameter( 'Excluded_I3VectorShieldHitRecord') self.AngularRecoName = self.GetParameter('AngularReco_I3Particle') self.EnergyRecoName = self.GetParameter('EnergyReco_I3Particle') self.Use_Laputop = self.GetParameter('Use_Laputop') self.LaputopParamsName = self.GetParameter('LaputopParamsName') self.RunMode = self.GetParameter('RunMode') self.Decimals = self.GetParameter('DecimalsForSanityCheck') if self.RunMode == 'GeneratePDF': self.OutputName = self.GetParameter('OutputFileName') self.binedges = self.GetParameter('BinEdges5D') self.distinct_regions_binedges = self.GetParameter( 'DistinctRegionsBinEdges3D') # make sure distinct regions binedges make sense if len(self.distinct_regions_binedges) == 0: # give the whole region as a distinct single region self.distinct_regions_binedges = [self.binedges[2:]] else: #check that each distinct region binedge is same shape as self.binedges i.e. for i in self.distinct_regions_binedges: if np.shape(i) != np.shape(self.binedges[2:]): print 'shape of self.binedges[2:] :', np.shape( self.binedges[2:]) print 'shape of self.distinct_regions_binedges', np.shape( self.distinct_regions_binedges) log_fatal( 'DistinctRegionBinEdges and BinEdges* not compatible' ) #check that joining all distinct regions gives total binedges check_distinct_regions_add_up_to_full( self.distinct_regions_binedges, self.binedges[2:], decimals=self.Decimals) self.labels = ['logE', 'cosZ', 'logQ', 'signedlogT', 'logRplusOne'] #creates the self.hist self._init_hist() elif self.RunMode == 'CalcLLHR': self.SigPDFInputName = self.GetParameter('SigPDFInputFileName') self.BkgPDFInputName = self.GetParameter('BkgPDFInputFileName') # this one should create self.bkg_hist, self.sig_hist, self.binedges, self.labels, self.distinct_regions_binedges self._load_PDF_from_file() self.SubtractEventFromPDF = self.GetParameter( 'SubtractEventFromPDF') self.objname = self.GetParameter('Output') return
def _fill(self,sample): h,edges=np.histogramdd(sample,self.binedges) if np.shape(h)!=np.shape(self.hist): log_fatal('initialized histogram and fill histogram dont match in shape') self.hist+= h self.n_events+=1 return
def Physics(self,frame): if self.RunMode=='GeneratePDF': self._GenPDFsPhysics(frame) elif self.RunMode=='CalcLLHR': self._CalcLLHRPhysics(frame) else: log_fatal('RunMode can only accept one these two inputs: GeneratePDF / CalcLLHR') self.PushFrame(frame) return
def __init__(self, encoder, inpdir, outfile): if encoder == 'ffmpeg': cmd = ['ffmpeg', '-i', '{0}/frame%08d.png'.format(inpdir)] cmd += ffmpeg_flags elif encoder == 'avconv': cmd = ['avconv', '-i', '{0}/frame%08d.png'.format(inpdir)] cmd += avconv_flags elif encoder == 'mencoder': cmd = ['mencoder', 'mf://{0}/*.png'.format(inpdir)] cmd += mencoder_flags else: i3logging.log_fatal("bad encoder argument: {0}".format(encoder)) cmd += [outfile] i3logging.log_debug('Encoding cmdline: ' + ' '.join(cmd)) self.proc = subprocess.Popen(cmd)
def _fill(self, sample): """ Pretty much all that is done during GeneratePDF mode. i.e. to fill up the histogram initialized during Configure. """ h, edges = np.histogramdd(sample=sample, bins=self.binedges) if np.shape(h) != np.shape(self.hist): log_fatal( 'initialized histogram and fill histogram dont match in shape') self.hist += h self.n_events += 1 #print self.n_events, np.sum(self.hist), np.sum(self.hist)/162., np.sum(h) #print RAM_test() return
def calc_LLHR(in_array, sig_hist, bkg_hist, binedges, distinct_regions_binedges, SubtractEventFromPDF): from icecube.icetray.i3logging import log_fatal, log_warn from llh_ratio_nd import get_slice_vector, log_likelihood_ratio d = {} d['llh_ratio'] = 0. d['n_extrapolations_sig_PDF'] = 0. d['n_extrapolations_bkg_PDF'] = 0. d['llh_sig'] = 0. d['llh_bkg'] = 0. d['isGood'] = 0. d['tanks_have_nans'] = 0. logE = in_array[0][0] coszen = in_array[0][1] # check if event logE and coszen lies within range of binedges if logE > binedges[0][-1] or logE < binedges[0][0]: return d if coszen > binedges[1][-1] or coszen < binedges[1][0]: return d # find the logE and coszen bins select those bins in sig/bkg pdfs logEbincenters = np.array((binedges[0][1:] + binedges[0][:-1]) / 2.) coszenbincenters = np.array((binedges[1][1:] + binedges[1][:-1]) / 2.) dE = np.absolute(logEbincenters - logE) Ebin = np.where(np.amin(dE) == dE)[0][0] dcZ = np.absolute(coszenbincenters - coszen) cZbin = np.where(np.amin(dcZ) == dcZ)[0][0] sig_hist = sig_hist[Ebin][cZbin] bkg_hist = bkg_hist[Ebin][cZbin] # select Q, T, R dimensions, generate event histogram in_array = (in_array.T[2:]).T binedges = binedges[2:] event_hist, temp = np.histogramdd(in_array, binedges) # store status if any of the q, t, r values have nans in this event if np.isnan(in_array[0][0]).any() or np.isnan( in_array[0][1]).any() or np.isnan(in_array[0][2]).any(): d['tanks_have_nans'] = 1. # subtract the event from the PDF if it was used for generating the PDF if SubtractEventFromPDF: if SubtractEventFromPDF == 'Sig': sig_hist = sig_hist - event_hist if (sig_hist < 0).any(): log_fatal('Event subtraction led to negative values') if SubtractEventFromPDF == 'Bkg': bkg_hist = bkg_hist - event_hist if (bkg_hist < 0).any(): log_fatal('Event subtraction led to negative values') # normalize histogram, obtain PDFs sig_pdf = sig_hist / np.sum(sig_hist) bkg_pdf = bkg_hist / np.sum(bkg_hist) # calculate llh ratio for each region separately and add it up # separate calculation is done to avoid one region influencing # extrapolated values of empty pixels in the PDF in another region llh_map_sig = np.zeros_like(sig_hist) llh_map_bkg = np.zeros_like(bkg_hist) d['isGood'] = 1. for region_edges in distinct_regions_binedges: # obtain slice vector for the region of the PDF region_range = [[i[0], i[-1]] for i in region_edges] slice_vector = get_slice_vector(binedges, region_range) temp = log_likelihood_ratio(heatmap1=sig_pdf[slice_vector], heatmap2=bkg_pdf[slice_vector], event_hist=event_hist[slice_vector]) d['llh_ratio'] += temp[0] # all the rest are debugging variables. some will be stored in I3VectorMap. # not storing any histograms as output. Just numbers. d['n_extrapolations_sig_PDF'] += temp[1] d['n_extrapolations_bkg_PDF'] += temp[2] d['llh_sig'] += temp[5] d['llh_bkg'] += temp[6] if np.isnan(temp[5]) or np.isnan(temp[6]): d['isGood'] = 0. # the following diagnostics are not being returned extrapolated_sig_PDF = temp[3] extrapolated_bkg_PDF = temp[4] llh_map_sig[slice_vector] = temp[7] llh_map_bkg[slice_vector] = temp[8] return d
def Physics(self, frame): #initiate output vectors hits = recclasses.I3VectorShieldHitRecord() unhits = recclasses.I3VectorShieldHitRecord() excluded = recclasses.I3VectorShieldHitRecord() not_unhit = [] # will keep track of hit+excluded doms #load reconstruction axis = frame[self.reco_name] rotation = to_shower_cs(axis) origin = np.array([[axis.pos.x], [axis.pos.y], [axis.pos.z]]) # populate hits vector for tankpulses in [self.slc_name, self.hlc_name]: pulses = dataclasses.I3RecoPulseSeriesMap.from_frame( frame, tankpulses) for omkey, pulse_items in pulses.iteritems(): if not omkey in self.geometry.omgeo: log_fatal("OM {om} not in geometry!".format(om=k)) if omkey[1] > 64 or omkey[1] < 61: log_fatal("OM {om} not an IceTop DOM!".format(om=k)) if len(pulse_items) != 1: log_fatal( 'only one pulse per omkey should be present. something might be fishy here.' ) not_unhit.append(omkey) for pulse in pulse_items: #calculate the tank's location/ time in shower frame ref # time is w.r.t. planar shower front arrival time at tank location position = self.geometry.omgeo[omkey].position det_cs_position = np.array([[position.x], [position.y], [position.z]]) shower_cs_position = rotation * (det_cs_position - origin) shower_cs_radius = np.sqrt(shower_cs_position[0]**2 + shower_cs_position[1]**2) time = pulse.time - float( axis.time - shower_cs_position[2] / I3Constants.c) #store all the pulse information in I3ShieldHitRecord and add it to hits vector new_pulse = recclasses.I3ShieldHitRecord() new_pulse.DOMkey = omkey new_pulse.charge = pulse.charge new_pulse.time_residual = time new_pulse.distance = np.float(shower_cs_radius) hits.append(new_pulse) # populate excluded vector tanklist = frame[self.excluded_name] for tank in tanklist: not_unhit.append(tank.default_omkey) new_pulse = recclasses.I3ShieldHitRecord() # new_pulse.DOMkey = tank new_pulse.DOMkey = tank.default_omkey new_pulse.charge = self.exc_fake_q new_pulse.time_residual = self.exc_fake_t position = self.geometry.omgeo[new_pulse.DOMkey].position det_cs_position = np.array([[position.x], [position.y], [position.z]]) shower_cs_position = rotation * (det_cs_position - origin) shower_cs_radius = np.sqrt(shower_cs_position[0]**2 + shower_cs_position[1]**2) new_pulse.distance = np.float(shower_cs_radius) excluded.append(new_pulse) #populate unhits vector stations = frame['I3Geometry'].stationgeo for station in stations: for tank in station.data(): is_unhit = 0 for dom in tank.omkey_list: if dom not in not_unhit: is_unhit += 1 if is_unhit == 2: new_pulse = recclasses.I3ShieldHitRecord() new_pulse.DOMkey = dom # pick the last one in the loop above new_pulse.charge = self.unhit_fake_q new_pulse.time_residual = self.unhit_fake_t position = self.geometry.omgeo[new_pulse.DOMkey].position det_cs_position = np.array([[position.x], [position.y], [position.z]]) shower_cs_position = rotation * (det_cs_position - origin) shower_cs_radius = np.sqrt(shower_cs_position[0]**2 + shower_cs_position[1]**2) new_pulse.distance = np.float(shower_cs_radius) unhits.append(new_pulse) #put the results to frame frame.Put(self.HitsName, hits) frame.Put(self.UnhitsName, unhits) frame.Put(self.ExcludedName, excluded) self.PushFrame(frame) return
def _CalcLLHRPhysics(self, frame): d = {} d['llh_ratio'] = 0. d['n_extrapolations_sig_PDF'] = 0. d['n_extrapolations_bkg_PDF'] = 0. d['llh_sig'] = 0. d['llh_bkg'] = 0. d['isGood'] = 0. # load event information in_array = self._create_in_array(frame) logE = in_array[0][0] coszen = in_array[0][1] # select Q, T, R dimensions, generate event histogram in_array = (in_array.T[2:]).T binedges = self.binedges[2:] event_hist, temp = np.histogramdd(in_array, binedges) # check if event logE and coszen lies within range of binedges if logE > self.binedges[0][-1] or logE < self.binedges[0][0]: frame.Put(self.objname, dataclasses.I3MapStringDouble(d)) return if coszen > self.binedges[1][-1] or coszen < self.binedges[1][0]: frame.Put(self.objname, dataclasses.I3MapStringDouble(d)) return # find the logE and coszen bins select those bins in sig/bkg pdfs logEbincenters = np.array( (self.binedges[0][1:] + self.binedges[0][:-1]) / 2.) coszenbincenters = np.array( (self.binedges[1][1:] + self.binedges[1][:-1]) / 2.) dE = np.absolute(logEbincenters - logE) Ebin = np.where(np.amin(dE) == dE)[0][0] dcZ = np.absolute(coszenbincenters - coszen) cZbin = np.where(np.amin(dcZ) == dcZ)[0][0] sig_hist = self.sig_hist[Ebin][cZbin] bkg_hist = self.bkg_hist[Ebin][cZbin] # subtract the event from the PDF if it was used for generating the PDF if self.SubtractEventFromPDF: if self.SubtractEventFromPDF == 'Sig': sig_hist = sig_hist - event_hist if (sig_hist < 0).any(): log_fatal('Event subtraction led to negative values') if self.SubtractEventFromPDF == 'Bkg': bkg_hist = bkg_hist - event_hist if (bkg_hist < 0).any(): log_fatal('Event subtraction led to negative values') # normalize histogram, obtain PDFs sig_pdf = sig_hist / np.sum(sig_hist) bkg_pdf = bkg_hist / np.sum(bkg_hist) # calculate llh ratio for each region separately and add it up # separate calculation is done to avoid one region influencing # extrapolated values of empty pixels in the PDF in another region llh_map_sig = np.zeros_like(sig_hist) llh_map_bkg = np.zeros_like(bkg_hist) d['isGood'] = 1. for region_edges in self.distinct_regions_binedges: # obtain slice vector for the region of the PDF region_range = [[i[0], i[-1]] for i in region_edges] slice_vector = get_slice_vector(binedges, region_range) temp = log_likelihood_ratio(heatmap1=sig_pdf[slice_vector], heatmap2=bkg_pdf[slice_vector], event_hist=event_hist[slice_vector]) d['llh_ratio'] += temp[0] # all the rest are debugging variables. some will be stored in I3VectorMap. # not storing any histograms as output. Just numbers. d['n_extrapolations_sig_PDF'] += temp[1] d['n_extrapolations_bkg_PDF'] += temp[2] d['llh_sig'] += temp[5] d['llh_bkg'] += temp[6] extrapolated_sig_PDF = temp[3] extrapolated_bkg_PDF = temp[4] llh_map_sig[slice_vector] = temp[7] llh_map_bkg[slice_vector] = temp[8] frame.Put(self.objname, dataclasses.I3MapStringDouble(d)) return
def _create_in_array(self, frame, time_transformation=signed_log): """ Create the IceTop specific input array that goes into GeneratePDF / CalcLLHR . This is method will need to be adapted for each analysis. """ if not self.Use_Laputop: En = np.log10(frame[self.EnergyRecoName].energy) else: En = frame[self.LaputopParamsName].value( recclasses.LaputopParameter.Log10_S125) ze = np.cos(frame[self.AngularRecoName].dir.zenith) hits = frame[self.HitsName] unhits = frame[self.UnhitsName] excluded = frame[self.ExcludedName] hits_t = time_transformation( np.array([hit.time_residual for hit in hits])) hits_q = np.log10(np.array([hit.charge for hit in hits])) hits_qunlog = np.array([hit.charge for hit in hits]) hits_r = log_plus_one(np.array([hit.distance for hit in hits])) hits_E = np.ones_like(hits_r) * En hits_z = np.ones_like(hits_r) * ze hits_doms = np.array([hit.DOMkey for hit in hits]) if np.isnan(hits_q).any(): select = np.isnan(hits_q) log_error('nan q doms in %s' % self.HitsName) log_error( zip(hits_q[select], hits_qunlog[select], hits_doms[select])) if np.isnan(hits_t).any(): select = np.isnan(hits_t) log_error('nan t doms in %s' % self.HitsName) log_error(zip(hits_t[select], hits_doms[select])) unhits_t = time_transformation( np.array([hit.time_residual for hit in unhits])) unhits_q = np.log10(np.array([hit.charge for hit in unhits])) unhits_r = log_plus_one(np.array([hit.distance for hit in unhits])) unhits_E = np.ones_like(unhits_r) * En unhits_z = np.ones_like(unhits_r) * ze excluded_t = time_transformation( np.array([hit.time_residual for hit in excluded])) excluded_q = np.log10(np.array([hit.charge for hit in excluded])) excluded_r = log_plus_one(np.array([hit.distance for hit in excluded])) excluded_E = np.ones_like(excluded_r) * En excluded_z = np.ones_like(excluded_r) * ze # ready data for entry to 5D hist t = np.concatenate((hits_t, unhits_t, excluded_t)) q = np.concatenate((hits_q, unhits_q, excluded_q)) r = np.concatenate((hits_r, unhits_r, excluded_r)) E = np.concatenate((hits_E, unhits_E, excluded_E)) z = np.concatenate((hits_z, unhits_z, excluded_z)) if len(t) != 162 or len(q) != 162 or len(r) != 162: log_error('N_t %s N_q %s N_r %s' % (len(t), len(q), len(r))) log_fatal('Total Tanks in Event not 162') if np.isnan(t).any() or np.isnan(q).any() or np.isnan(r).any(): #print 't',t #print 'q',q #print 'r',r log_warn( 'signed_time/logq/logr have nans, logs125%.2f coszen%.2f' % (En, ze)) in_array = np.vstack([E, z, q, t, r]).T return in_array