def DoLLReconstructions(tray, Pulses, suffix):
tray.AddSegment(linefit.simple,
suffix + "_imprv_LF",
inputResponse=Pulses,
fitName="linefit_final" + suffix,
If=which_split(split_name='InIceSplit'))
tray.AddSegment(SPE,
'SPE' + suffix,
Pulses=Pulses,
suffix=suffix,
LineFit='LineFit',
SPEFitSingle='SPEFitSingle',
SPEFit='SPEFit2',
SPEFitCramerRao='SPEFit2CramerRao',
N_iter=2)
tray.AddSegment(MPE,
'MPE' + suffix,
Pulses=Pulses,
Seed='SPEFit2',
suffix=suffix,
MPEFit='MPEFit',
MPEFitCramerRao='MPEFitCramerRao')
"ReducedCountMaker",
Streams=[icetray.I3Frame.DAQ])
tray.AddModule("HypoFrameCreator",
"HypoFrameCreator",
SplitName=SplitName,
HypoName="hypoframe",
RecoMapName=SplitPulses,
MaxTimeSeparation=3000. * I3Units.ns)
from icecube import linefit, lilliput
tray.AddSegment(linefit.simple,
'LineFit',
inputResponse=SplitPulses,
fitName=FitName,
If=which_split(SplitName) | which_split('hypoframe'))
#=== A bunch of tester modules ===
#allow a unique name to the modules
if (test):
redo_org = 'redo'
else:
redo_org = 'org'
tray.AddModule("AfterpulseTester",
redo_org + "Afterpulse",
SplitName=SplitName,
HypoName="hypoframe",
RecoMapName=SplitPulses,
QTotFraction=.1,
TimeOffset=3.E3 * I3Units.ns,
def OnlineFilter(tray,
name,
simulation=False,
decode=False,
If=lambda f: True,
SplineRecoAmplitudeTable=None,
SplineRecoTimingTable=None,
PathToCramerRaoTable=None,
GFUBDTUpPath=None,
GFUBDTDownPath=None,
sdstarchive=False,
slop_split_enabled=True,
vemcal_enabled=True,
gfu_enabled=True,
needs_wavedeform_spe_corr=True,
alert_followup=True,
alert_followup_omit_GCD_diff=False,
alert_followup_base_GCD_path="",
alert_followup_base_GCD_filename=None,
forceOnlineL2BadDOMList=None,
ic79_geometry=False):
from icecube.phys_services.which_split import which_split
# import all the actual worker segments....
from icecube.filterscripts.baseproc import BaseProcessing
from icecube.filterscripts.minbiasfilters import MinBiasFilters
from icecube.filterscripts.minbiasfilters import ScintMinBiasFilters
from icecube.filterscripts.dstfilter import DSTFilter
from icecube.filterscripts.muonfilter import MuonFilter
from icecube.filterscripts.cosmicrayfilter import CosmicRayFilter
from icecube.filterscripts.cascadefilter import CascadeFilter
from icecube.filterscripts.onlinel2filter import OnlineL2Filter
from icecube.filterscripts.shadowfilter import ShadowFilter
from icecube.filterscripts.deepcorefilter import DeepCoreFilter
from icecube.filterscripts.fssfilter import FSSFilter
from icecube.filterscripts.ehefilter import EHEFilter
from icecube.filterscripts.lowupfilter import LowUpFilter
from icecube.filterscripts.veffilter import VEFFilter
from icecube.filterscripts.slopfilter import SLOPFilter
from icecube.filterscripts.slopfilter import SLOPSplitter
from icecube.filterscripts.fixedratefilter import FixedRateTrigFilter
from icecube.filterscripts.hesefilter import HeseFilter
from icecube.filterscripts.highqfilter import HighQFilter
from icecube.filterscripts.mesefilter import MeseFilter
from icecube.filterscripts.ehealertfilter import EHEAlertFilter
from icecube.filterscripts.monopolefilter import MonopoleFilter
from icecube.filterscripts.icetop2station import IceTopTwoStationFilter
from icecube.filterscripts.estresfilter import ESTReSFilter
from icecube.filterscripts.iceactsmtfilter import IceActTrigFilter
from icecube.filterscripts.alerteventfollowup import AlertEventFollowup
from icecube.filterscripts.grecofilter import GRECOOnlineFilter
# Get the spline paths sorted.
# This is used by Online L2 and ESTReS
if (SplineRecoAmplitudeTable is None) or (SplineRecoTimingTable is None):
domain = filter_globals.getDomain()
if (domain == filter_globals.Domain.SPS) or (
domain == filter_globals.Domain.SPTS):
SplineDir = '/opt/cvmfs/current/data/photon-tables/splines/'
SplineRecoAmplitudeTable = os.path.join(
SplineDir, 'InfBareMu_mie_abs_z20a10.fits')
SplineRecoTimingTable = os.path.join(
SplineDir, 'InfBareMu_mie_prob_z20a10.fits')
elif domain == filter_globals.Domain.ACCESS:
SplineDir = '/opt/cvmfs/current/data/photon-tables/splines/'
SplineRecoAmplitudeTable = os.path.join(
SplineDir, 'InfBareMu_mie_abs_z20a10.fits')
SplineRecoTimingTable = os.path.join(
SplineDir, 'InfBareMu_mie_prob_z20a10.fits')
else:
SplineDir = "/cvmfs/icecube.opensciencegrid.org/data/photon-tables/splines/"
SplineRecoAmplitudeTable = os.path.join(
SplineDir, 'InfBareMu_mie_abs_z20a10_V2.fits')
SplineRecoTimingTable = os.path.join(
SplineDir, 'InfBareMu_mie_prob_z20a10_V2.fits')
# Create a SeededRT configuration object with the standard RT settings.
# This object will be used by all the different SeededRT modules, i.e. the
# modules use the same causial space and time conditions, but can use
# different seed algorithms.
from icecube.STTools.seededRT.configuration_services import I3DOMLinkSeededRTConfigurationService
seededRTConfig = I3DOMLinkSeededRTConfigurationService(
ic_ic_RTRadius=150.0 * I3Units.m,
ic_ic_RTTime=1000.0 * I3Units.ns,
treat_string_36_as_deepcore=False,
useDustlayerCorrection=False,
allowSelfCoincidence=True)
## base processing requires: GCD and frames being fed in by reader or Inlet
## base processing include: decoding, TriggerCheck, Bad Dom cleaning, calibrators,
## Feature extraction, pulse cleaning (seeded RT, and Time Window),
## PoleMuonLineit, PoleMuonLlh, Cuts module and Mue on PoleMuonLlh
if sdstarchive:
tray.AddSegment(BaseProcessing,
"BaseProc",
pulses=filter_globals.CleanedMuonPulses,
decode=decode,
simulation=False,
needs_calibration=False,
needs_superdst=False,
needs_maskmaker=True,
do_slop=slop_split_enabled,
needs_trimmer=False,
seededRTConfig=seededRTConfig)
else:
tray.AddSegment(BaseProcessing,
"BaseProc",
pulses=filter_globals.CleanedMuonPulses,
decode=decode,
simulation=simulation,
do_slop=slop_split_enabled,
seededRTConfig=seededRTConfig,
needs_wavedeform_spe_corr=needs_wavedeform_spe_corr)
## Todo: Check with IceTop group has this changed?
## Note: simulation doesn't run VEMCAL
if vemcal_enabled:
from icecube.filterscripts.vemcal import IceTopVEMCal
tray.AddSegment(IceTopVEMCal, "VEMCALStuff")
## Filters that use the InIce Trigger splitting
tray.AddSegment(MuonFilter,
"MuonFilter",
pulses=filter_globals.CleanedMuonPulses,
If=which_split(split_name=filter_globals.InIceSplitter))
## In the FSS segment the "pulses" are used for finiteReco, after customized hit
## cleaning which is not optimized for track fitting but rather for vetoing
## downgoing muons in top & side layers. So: do *not* provide pulses that are
## already e.g. SRT cleaned, such as CleanedMuonPulses.
tray.AddSegment(FSSFilter,
"FSSFilter",
pulses=filter_globals.SplitUncleanedInIcePulses,
If=which_split(split_name=filter_globals.InIceSplitter),
ic79_geometry=ic79_geometry)
tray.AddSegment(CascadeFilter,
"CascadeFilter",
pulses=filter_globals.CleanedMuonPulses,
muon_llhfit_name=filter_globals.muon_llhfit,
If=which_split(split_name=filter_globals.InIceSplitter))
tray.AddSegment(VEFFilter,
"VEF",
pulses=filter_globals.CleanedMuonPulses,
If=which_split(split_name=filter_globals.InIceSplitter))
tray.AddSegment(LowUpFilter,
"LowUpFilter",
If=which_split(split_name=filter_globals.InIceSplitter))
# High Q filter
tray.AddSegment(HighQFilter,
"HighQFilter",
pulses=filter_globals.SplitUncleanedInIcePulses,
If=(which_split(split_name=filter_globals.InIceSplitter)))
# HESE veto (VHESelfVeto)
tray.AddSegment(HeseFilter,
"HeseFilter",
pulses=filter_globals.SplitUncleanedInIcePulses,
If=(which_split(split_name=filter_globals.InIceSplitter)))
# MESE veto (Jakob van Santen's veto)
tray.AddSegment(MeseFilter,
"MeseFilter",
pulses=filter_globals.SplitUncleanedInIcePulses,
If=(which_split(split_name=filter_globals.InIceSplitter)))
# OnlineL2, used by HESE and GFU
tray.AddSegment(OnlineL2Filter,
"OnlineL2",
pulses=filter_globals.CleanedMuonPulses,
linefit_name=filter_globals.muon_linefit,
llhfit_name=filter_globals.muon_llhfit,
SplineRecoAmplitudeTable=SplineRecoAmplitudeTable,
SplineRecoTimingTable=SplineRecoTimingTable,
PathToCramerRaoTable=PathToCramerRaoTable,
forceOnlineL2BadDOMList=forceOnlineL2BadDOMList,
If=which_split(split_name=filter_globals.InIceSplitter))
if (simulation):
# sim data
tray.AddSegment(
ShadowFilter,
"MoonAndSun",
mcseed=424242,
If=which_split(split_name=filter_globals.InIceSplitter))
else:
# exp data
tray.AddSegment(
ShadowFilter,
"MoonAndSun",
If=which_split(split_name=filter_globals.InIceSplitter))
tray.AddSegment(ESTReSFilter,
"ESTReSFilter",
pulsesname=filter_globals.SplitUncleanedInIcePulses,
base_processing_fit=filter_globals.muon_llhfit,
SplineRecoAmplitudeTable=SplineRecoAmplitudeTable,
SplineRecoTimingTable=SplineRecoTimingTable,
If=which_split(split_name=filter_globals.InIceSplitter))
tray.AddSegment(DeepCoreFilter,
"DeepCoreFilter",
pulses=filter_globals.SplitUncleanedInIcePulses,
seededRTConfig=seededRTConfig,
If=which_split(split_name=filter_globals.InIceSplitter))
tray.AddSegment(GRECOOnlineFilter,
"GRECOOnlineFilter",
uncleaned_pulses=filter_globals.SplitUncleanedInIcePulses,
If=which_split(split_name=filter_globals.InIceSplitter))
#EHE, now on the InIce!
tray.AddSegment(EHEFilter,
"EHEFilter",
If=which_split(split_name=filter_globals.InIceSplitter))
## Filters on the Null split
tray.AddSegment(MinBiasFilters,
"MinBias",
If=which_split(split_name=filter_globals.NullSplitter))
tray.AddSegment(FixedRateTrigFilter,
"FixedRate",
If=which_split(split_name=filter_globals.NullSplitter))
tray.AddSegment(ScintMinBiasFilters,
"ScintMinBias",
If=which_split(split_name=filter_globals.NullSplitter))
tray.AddSegment(IceActTrigFilter,
"IceActFilters",
If=which_split(split_name=filter_globals.NullSplitter))
tray.AddSegment(DSTFilter,
"DSTFilter",
dstname=filter_globals.dst,
pulses=filter_globals.CleanedMuonPulses,
If=which_split(split_name=filter_globals.InIceSplitter))
## SLOP filter
if slop_split_enabled:
tray.AddSegment(SLOPFilter,
"SLOPFilter",
use_pulses=sdstarchive,
If=which_split(split_name=filter_globals.SLOPSplitter))
## Filters on the CR split
tray.AddSegment(CosmicRayFilter,
"CosmicRayFilter",
Pulses=filter_globals.CleanedHLCTankPulses,
If=which_split(split_name=filter_globals.IceTopSplitter))
tray.AddSegment(IceTopTwoStationFilter,
"TwoStationFilter",
If=which_split(split_name=filter_globals.IceTopSplitter))
## Gamma-Ray Follow-Up
if gfu_enabled:
from icecube.filterscripts.gfufilter import GammaFollowUp
tray.AddSegment(
GammaFollowUp,
"GammaFollowUp",
OnlineL2SegmentName="OnlineL2",
BDTUpFile=GFUBDTUpPath,
BDTDownFile=GFUBDTDownPath,
angular_error=True,
If=which_split(split_name=filter_globals.InIceSplitter))
## EHE Online Alert Follow-up
tray.AddSegment(EHEAlertFilter,
"EHEAlertFilter",
If=which_split(split_name=filter_globals.InIceSplitter))
## Monopole filter
tray.AddSegment(MonopoleFilter,
"MonopoleFilter",
pulses=filter_globals.SplitUncleanedInIcePulses,
seededRTConfig=seededRTConfig,
If=which_split(split_name=filter_globals.InIceSplitter))
# Alert followup
if alert_followup:
tray.AddSegment(
AlertEventFollowup,
"AlertFollowup",
omit_GCD_diff=alert_followup_omit_GCD_diff,
base_GCD_path=alert_followup_base_GCD_path,
base_GCD_filename=alert_followup_base_GCD_filename,
If=(which_split(split_name=filter_globals.InIceSplitter)))
return False
return icetray.pypick(f)
tray = I3Tray()
tray.Add(split_test_frame_generator, "bogogenesis")
#tray.Add("Dump","truck")
tray.Add(
schizzoparanoia,
"null",
ShouldRunOn=set([('null', 0)]),
If=which_split('null'),
)
tray.Add(
schizzoparanoia,
"foo_bar",
ShouldRunOn=set([('foo', i)
for i in range(Nfoo)] + [('bar', j)
for j in range(Nbar)]),
If=which_split('foo') | which_split('bar'),
)
tray.Add(
schizzoparanoia,
"also_foo_bar",
ShouldRunOn=set([('foo', i)
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
if int(yaml.__version__[0]) < 5:
# backwards compatibility for yaml versions before version 5
cfg = yaml.load(stream)
else:
cfg = yaml.full_load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
infile = cfg['infile_pattern'].format(**cfg)
infile = infile.replace(' ', '0')
infile = infile.replace('Level0.{}'.format(cfg['previous_step']),
'Level0.{}'.format(cfg['previous_step'] % 10))
infile = infile.replace('2012_pass2', 'pass2')
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace('Level0.{}'.format(cfg['step']),
'Level0.{}'.format(cfg['step'] % 10))
outfile = outfile.replace(' ', '0')
outfile = outfile.replace('2012_pass2', 'pass2')
print('Outfile != $FINAL_OUT clean up for crashed scripts not possible!')
# collect keys which are to be kept in a addition to the official keep keys
# of the standard L1 and L2 processing
if 'additional_keep_keys' in cfg:
additional_keep_keys = cfg['additional_keep_keys'] + muongun_keys
else:
additional_keep_keys = muongun_keys
additional_keep_keys += [
'BiasedMuonWeighter', 'BiasedMuonCorridorWeighter',
'BiasedMESCHotspotWeighter', 'BiasedSimulationWeight',
'PROPOSALStorm', 'PROPOSALStormUniformRanges',
'MCVetoMuonInjectionInfo', 'MMCTrackListVetoMuon',
'CombinedMuonVetoI3MCTree', 'I3MCTreeVetoMuon',
'I3MCTreeVetoMuon_preMuonProp',
'I3MCTreeVetoMuon_preMuonProp_RNGState',
]
tray = I3Tray()
"""The main L1 script"""
tray.AddModule('I3Reader',
'i3 reader',
FilenameList=[cfg['gcd_pass2'], infile])
# run online filters
online_kwargs = {}
if SPLINE_TABLES:
online_kwargs.update({
'SplineRecoAmplitudeTable': os.path.join(
SPLINE_TABLES, 'InfBareMu_mie_abs_z20a10.fits'),
'SplineRecoTimingTable': os.path.join(
SPLINE_TABLES, 'InfBareMu_mie_prob_z20a10.fits'),
# 'alert_followup_base_GCD_filename': cfg['gcd_pass2'],
})
if cfg['L1_pass2_run_gfu'] is not None:
online_kwargs['gfu_enabled'] = cfg['L1_pass2_run_gfu']
if 'L1_needs_wavedeform_spe_corr' not in cfg:
cfg['L1_needs_wavedeform_spe_corr'] = False
tray.AddSegment(OnlineFilter, "OnlineFilter",
decode=False, simulation=True,
vemcal_enabled=False,
alert_followup=False,
needs_wavedeform_spe_corr=cfg[
'L1_needs_wavedeform_spe_corr'],
**online_kwargs
)
# make random service
_, seed = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=0)
filter_mask_randoms = phys_services.I3GSLRandomService(seed)
# override MinBias Prescale
filterconfigs = filter_globals.filter_pairs + filter_globals.sdst_pairs
print(cfg['L1_min_bias_prescale'])
if cfg['L1_min_bias_prescale']:
for i, filtertuple in enumerate(filterconfigs):
if filtertuple[0] == filter_globals.FilterMinBias:
del filterconfigs[i]
filterconfigs.append((filtertuple[0],
cfg['L1_min_bias_prescale']))
break
print(filterconfigs)
# Generate filter Masks for all P frames
tray.AddModule(filter_tools.FilterMaskMaker,
"MakeFilterMasks",
OutputMaskName=filter_globals.filter_mask,
FilterConfigs=filterconfigs,
RandomService=filter_mask_randoms)
# Merge the FilterMasks
tray.AddModule("OrPframeFilterMasks",
"make_q_filtermask",
InputName=filter_globals.filter_mask,
OutputName=filter_globals.qfilter_mask)
# Q+P frame specific keep module needs to go first, as KeepFromSubstram
# will rename things, let's rename post keep.
def is_Q(frame):
return frame.Stop == frame.DAQ
simulation_keeps = [
'BackgroundI3MCTree',
'BackgroundI3MCTreePEcounts',
'BackgroundI3MCPESeriesMap',
'BackgroundI3MCTree_preMuonProp',
'BackgroundI3MCTree_preMuonProp_RNGState',
'BackgroundMMCTrackList',
'BeaconLaunches',
'CorsikaInteractionHeight',
'CorsikaWeightMap',
'EventProperties',
'GenerationSpec',
'I3LinearizedMCTree',
'I3MCTree',
'I3MCTreePEcounts',
'I3MCTree_preMuonProp',
'I3MCTree_preMuonProp_RNGState',
'I3MCPESeriesMap',
'I3MCPESeriesMapWithoutNoise',
'I3MCPESeriesMapParticleIDMap',
'I3MCPulseSeriesMap',
'I3MCPulseSeriesMapParticleIDMap',
'I3MCPulseSeriesMapPrimaryIDMap',
'I3MCWeightDict',
'LeptonInjectorProperties',
'MCHitSeriesMap',
'MCPrimary',
'MCPrimaryInfo',
'MMCTrackList',
'PolyplopiaInfo',
'PolyplopiaPrimary',
'RNGState',
'SignalI3MCPEs',
'SimTrimmer', # for SimTrimmer flag
'TimeShift', # the time shift amount
'WIMP_params', # Wimp-sim
'noise_weight', # weights for noise-only vuvuzela simulations
'I3GENIEResultDict' # weight informaition for GENIE simulations
] + additional_keep_keys
keep_before_merge = filter_globals.q_frame_keeps + [
'InIceDSTPulses', # keep DST pulse masks
'IceTopDSTPulses',
'CalibratedWaveformRange', # keep calibration info
'UncleanedInIcePulsesTimeRange',
'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange',
'CalibrationErrata',
'SaturationWindows',
'InIceRawData', # keep raw data for now
'IceTopRawData',
] + simulation_keeps
tray.AddModule("Keep", "keep_before_merge",
keys=keep_before_merge,
If=is_Q)
# second set of prekeeps, conditional on filter content, based on newly
# created Qfiltermask
# Determine if we should apply harsh keep for events that failed to pass
# any filter
# Note: excluding the sdst_streams entries
tray.AddModule("I3IcePickModule<FilterMaskFilter>", "filterMaskCheckAll",
FilterNameList=filter_globals.filter_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedAnyFilter",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def do_save_just_superdst(frame):
if frame.Has("PassedAnyFilter"):
if not frame["PassedAnyFilter"].value:
return True # <- Event failed to pass any filter.
else:
return False # <- Event passed some filter
else:
icetray.logging.log_error("Failed to find key frame Bool!!")
return False
keep_only_superdsts = filter_globals.keep_nofilterpass + [
'PassedAnyFilter',
'InIceDSTPulses',
'IceTopDSTPulses',
'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange',
'RNGState'] + simulation_keeps
tray.AddModule("Keep", "KeepOnlySuperDSTs",
keys=keep_only_superdsts,
If=do_save_just_superdst)
# Now clean up the events that not even the SuperDST filters passed on
tray.AddModule("I3IcePickModule<FilterMaskFilter>", "filterMaskCheckSDST",
FilterNameList=filter_globals.sdst_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedKeepSuperDSTOnly",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def dont_save_superdst(frame):
if frame.Has("PassedKeepSuperDSTOnly") and \
frame.Has("PassedAnyFilter"):
if frame["PassedAnyFilter"].value:
return False # <- these passed a regular filter, keeper
elif not frame["PassedKeepSuperDSTOnly"].value:
return True # <- Event failed to pass SDST filter.
else:
return False # <- Event passed some SDST filter
else:
icetray.logging.log_error("Failed to find key frame Bool!!")
return False
# backward compatibility
if 'L1_keep_untriggered' in cfg and cfg['L1_keep_untriggered']:
discard_substream_and_keys = False
else:
discard_substream_and_keys = True
if discard_substream_and_keys:
tray.AddModule(
"Keep", "KeepOnlyDSTs",
keys=filter_globals.keep_dst_only + [
"PassedAnyFilter",
"PassedKeepSuperDSTOnly",
filter_globals.eventheader] + additional_keep_keys,
If=dont_save_superdst)
# Frames should now contain only what is needed. now flatten,
# write/send to server
# Squish P frames back to single Q frame, one for each split:
tray.AddModule("KeepFromSubstream", "null_stream",
StreamName=filter_globals.NullSplitter,
KeepKeys=filter_globals.null_split_keeps)
in_ice_keeps = filter_globals.inice_split_keeps + \
filter_globals.onlinel2filter_keeps
in_ice_keeps = in_ice_keeps + [
'I3EventHeader',
'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'TriggerSplitterLaunchWindow',
'I3TriggerHierarchy',
'GCFilter_GCFilterMJD'] + additional_keep_keys
tray.AddModule("Keep", "inice_keeps",
keys=in_ice_keeps,
If=which_split(split_name=filter_globals.InIceSplitter),)
tray.AddModule("KeepFromSubstream", "icetop_split_stream",
StreamName=filter_globals.IceTopSplitter,
KeepKeys=filter_globals.icetop_split_keeps)
# Apply small keep list (SuperDST/SmallTrig/DST/FilterMask for non-filter
# passers
# Remove I3DAQData object for events not passing one of the
# 'filters_keeping_allraw'
tray.AddModule("I3IcePickModule<FilterMaskFilter>", "filterMaskCheck",
FilterNameList=filter_globals.filters_keeping_allraw,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedConventional",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
# Clean out the Raw Data when not passing conventional filter
def I3RawDataCleaner(frame):
if not (('PassedConventional' in frame and
frame['PassedConventional'].value == True) or
('SimTrimmer' in frame and
frame['SimTrimmer'].value == True)):
frame.Delete('InIceRawData')
frame.Delete('IceTopRawData')
tray.AddModule(I3RawDataCleaner,
"CleanErrataForConventional",
Streams=[icetray.I3Frame.DAQ])
tray.AddModule("I3Writer", "EventWriter",
filename=outfile,
Streams=[icetray.I3Frame.DAQ,
icetray.I3Frame.Physics,
icetray.I3Frame.TrayInfo,
icetray.I3Frame.Simulation,
icetray.I3Frame.Stream('m'),
icetray.I3Frame.Stream('M')])
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
cfg = yaml.load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
click.echo('Keep all OnlineL2: {}'.format(cfg['OnlineL2_keep_all_L2']))
click.echo('Keep time residuals: {}'.format(
cfg['OnlineL2_keep_time_residuals']))
infile = cfg['infile_pattern'].format(**cfg)
infile = infile.replace(' ', '0')
infile = infile.replace('Level0.{}'.format(cfg['previous_step']), 'Level2')
infile = infile.replace('Level0.{}'.format(cfg['previous_step']),
'Level0.{}'.format(cfg['previous_step'] % 10))
infile = infile.replace('2012_pass2', '2012')
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace('Level0.{}'.format(cfg['step']), '2017OnlineL2')
outfile = outfile.replace(' ', '0')
outfile = outfile.replace('2012_pass2', '2012')
print('Outfile != $FINAL_OUT clean up for crashed scripts not possible!')
# build tray
tray = I3Tray()
tray.context['I3FileStager'] = dataio.get_stagers()
tray.Add('I3Reader',
FilenameList=[cfg['gcd_pass2'], infile],
SkipKeys=['I3MCTree'] if 'corsika' in infile.lower() else [])
# drop exisiting P-Frames (will do our own splitting later)
tray.Add(lambda f: False, Streams=[icetray.I3Frame.Physics])
############################################################################
# the following modules repeat what is done in the base processing at Pole #
############################################################################
# resplit Q frame
icetray.load('trigger-splitter', False)
tray.AddModule('I3TriggerSplitter',
filter_globals.InIceSplitter,
TrigHierName='DSTTriggers',
TriggerConfigIDs=[
filter_globals.deepcoreconfigid,
filter_globals.inicesmtconfigid,
filter_globals.inicestringconfigid,
filter_globals.volumetriggerconfigid
],
SubEventStreamName=filter_globals.InIceSplitter,
InputResponses=['InIceDSTPulses'],
OutputResponses=[filter_globals.SplitUncleanedInIcePulses],
WriteTimeWindow=True)
# evaluate TriggerHierarchy
tray.AddModule(
"TriggerCheck_13",
"BaseProc_Trigchecker",
I3TriggerHierarchy=filter_globals.triggerhierarchy,
InIceSMTFlag=filter_globals.inicesmttriggered,
IceTopSMTFlag=filter_globals.icetopsmttriggered,
InIceStringFlag=filter_globals.inicestringtriggered,
DeepCoreSMTFlag=filter_globals.deepcoresmttriggered,
DeepCoreSMTConfigID=filter_globals.deepcoreconfigid,
VolumeTriggerFlag=filter_globals.volumetrigtriggered,
SlowParticleFlag=filter_globals.slowparticletriggered,
FixedRateTriggerFlag=filter_globals.fixedratetriggered,
)
# run SRT and TW Cleaning from the Base Processing
from icecube.STTools.seededRT.configuration_services import I3DOMLinkSeededRTConfigurationService
seededRTConfig = I3DOMLinkSeededRTConfigurationService(
ic_ic_RTRadius=150.0 * I3Units.m,
ic_ic_RTTime=1000.0 * I3Units.ns,
treat_string_36_as_deepcore=False,
useDustlayerCorrection=False,
allowSelfCoincidence=True)
tray.AddModule(
'I3SeededRTCleaning_RecoPulseMask_Module',
'BaseProc_seededrt',
InputHitSeriesMapName=filter_globals.SplitUncleanedInIcePulses,
OutputHitSeriesMapName=filter_globals.SplitRTCleanedInIcePulses,
STConfigService=seededRTConfig,
SeedProcedure='HLCCoreHits',
NHitsThreshold=2,
MaxNIterations=3,
Streams=[icetray.I3Frame.Physics],
If=which_split(split_name=filter_globals.InIceSplitter))
tray.AddModule("I3TimeWindowCleaning<I3RecoPulse>",
"TimeWindowCleaning",
InputResponse=filter_globals.SplitRTCleanedInIcePulses,
OutputResponse=filter_globals.CleanedMuonPulses,
TimeWindow=6000 * I3Units.ns,
If=which_split(split_name=filter_globals.InIceSplitter))
tray.AddSegment(linefit.simple,
"BaseProc_imprv_LF",
inputResponse=filter_globals.CleanedMuonPulses,
fitName=filter_globals.muon_linefit,
If=which_split(split_name=filter_globals.InIceSplitter))
# Muon LLH SimpleFitter from GulliverSuite with LineFit seed.
tray.AddSegment(lilliput.segments.I3SinglePandelFitter,
filter_globals.muon_llhfit,
seeds=[filter_globals.muon_linefit],
pulses=filter_globals.CleanedMuonPulses,
If=which_split(split_name=filter_globals.InIceSplitter))
# run MuonFilter
tray.Add(MuonFilter,
'MuonFilter',
pulses=filter_globals.CleanedMuonPulses,
If=which_split(split_name=filter_globals.InIceSplitter))
tray.AddModule(
"I3FirstPulsifier",
"BaseProc_first-pulsify",
InputPulseSeriesMapName=filter_globals.CleanedMuonPulses,
OutputPulseSeriesMapName='FirstPulseMuonPulses',
KeepOnlyFirstCharge=False, # default
UseMask=False, # default
If=which_split(split_name=filter_globals.InIceSplitter))
# discard events not passing the MuonFilter
tray.Add(lambda f: f.Has(filter_globals.MuonFilter) and f[
filter_globals.MuonFilter].value)
# run OnlineL2 filter
tray.Add(TimerStart,
timerName='OnlineL2',
If=which_split(split_name=filter_globals.InIceSplitter))
tray.AddSegment(OnlineL2Filter,
"OnlineL2",
If=which_split(split_name=filter_globals.InIceSplitter))
tray.Add(TimerStop, timerName='OnlineL2')
# discard events not passing the OnlineL2 filter
tray.Add(lambda f: f.Has(filter_globals.OnlineL2Filter) and f[
filter_globals.OnlineL2Filter].value)
# run GFU filter
tray.Add(TimerStart, timerName='GFU')
tray.AddSegment(GammaFollowUp,
"GFU",
OnlineL2SegmentName="OnlineL2",
KeepDetails=cfg['OnlineL2_keep_time_residuals'],
angular_error=True)
tray.Add(TimerStop, timerName='GFU')
# discard events not passing the GFU filter
if not cfg['OnlineL2_keep_all_L2']:
tray.Add(lambda f: f.Has(filter_globals.GFUFilter) and f[
filter_globals.GFUFilter].value)
# in this case, also run splineMPE with maximum settings for comparison
TEestis = [
'OnlineL2_SplineMPE_TruncatedEnergy_AllDOMS_Muon',
'OnlineL2_SplineMPE_TruncatedEnergy_DOMS_Muon',
'OnlineL2_SplineMPE_TruncatedEnergy_AllBINS_Muon',
'OnlineL2_SplineMPE_TruncatedEnergy_BINS_Muon',
'OnlineL2_SplineMPE_TruncatedEnergy_ORIG_Muon'
]
tray.Add(CustomSplineMPE,
'SplineMPEmax',
configuration='max',
pulses='OnlineL2_CleanedMuonPulses',
trackSeeds=['OnlineL2_SplineMPE'],
enEstis=TEestis,
paraboloid=True)
# For MC weighting, keep the neutrino primary.
if 'corsika' not in infile.lower():
# Some CORSIKA files have I3MCTree objects much larger than 100 MB.
# Loading them takes too long... instead use CorsikaWeightMap.PrimaryEnergy / PrimaryType for weighting.
tray.AddModule(get_weighted_primary,
'get_weighted_primary',
MCPrimary='I3MCPrimary')
# For MC studies, store information about the muon from CC interaction
if 'neutrino-generator' in infile.lower():
# store muon intersection points
tray.Add(AddMuon)
tray.Add(AddMuonIntersection)
# store deposited energy in detector
tray.Add(AddDepositedEnergy)
tray.AddModule("I3Writer",
"EventWriter",
filename=outfile,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation
],
DropOrphanStreams=[icetray.I3Frame.DAQ])
tray.AddModule("TrashCan", "the can")
tray.Execute()
del tray
ic_ic_RTRadius=200.0 * I3Units.m,
ic_ic_RTTime=750.0 * I3Units.ns,
treat_string_36_as_deepcore=False,
useDustlayerCorrection=False,
allowSelfCoincidence=True)
tray.AddModule('I3SeededRTCleaning_RecoPulseMask_Module',
'North_seededrt_1',
InputHitSeriesMapName=hit_series,
OutputHitSeriesMapName='SRTInIcePulses_200_750_1',
STConfigService=seededRTConfig1,
SeedProcedure='HLCCoreHits',
NHitsThreshold=2,
MaxNIterations=1,
Streams=[icetray.I3Frame.Physics],
If=which_split(split_name='InIceSplit') &
(lambda f: (deepcore_wg(f) or wimp_wg(f) or muon_wg(f) or
cascade_wg(f) or ehe_wg(f) or fss_wg(f))))
seededRTConfig2 = I3DOMLinkSeededRTConfigurationService(
ic_ic_RTRadius=90.0 * I3Units.m,
ic_ic_RTTime=700.0 * I3Units.ns,
treat_string_36_as_deepcore=False,
useDustlayerCorrection=False,
allowSelfCoincidence=True)
tray.AddModule('I3SeededRTCleaning_RecoPulseMask_Module',
'North_seededrt_2',
InputHitSeriesMapName=hit_series,
OutputHitSeriesMapName='SRTInIcePulses_90_700_1',
STConfigService=seededRTConfig2,
def Split_Recombine(tray, name,
OrgPulses = "", # NOTE Define the name of the unaltered Pulses in the Q-frame
SplitName = "", # NOTE Define the name of the subevent stream that you like to create
SplitPulses = ""): # NOTE Define how the split Pulses should be named
# NOTE Use a generic event Splitter which works on Q-frames and writes P frames with the output being the SplitPulses
# NOTE Do not forget to save the SplitCount to the Q-frame in the format <I3Int>(SplitName+"SplitCount")
# from icecube import IceHive
# tray.AddSegment(IceHive.Split_and_Recombine,"IceHive_Splitting")
from icecube import CoincSuite
# NOTE Create a complement object <I3Int>(SplitName+"ReducedName") in the Q frame keeping trace of the number of frames which have been recombined
tray.AddModule(lambda f: f.Put(SplitName+"ReducedCount", icetray.I3Int(0)), "ReducedCountMaker",
Streams = [icetray.I3Frame.DAQ])
# NOTE Discard frames that you do not want to concider for recombinations early on.
# NOTE every discarded frame is accounted for by increasing the 'ReducedCount'
tray.AddModule("AfterpulseDiscard", "AfterpulseDiscard",
SplitName = SplitName, #NOTE Run on the SplitFrames; this parameter should be set for all modules
RecoMapName = SplitPulses, #NOTE use the SplitPulses; this parameter should be set for all modules
QTotFraction = .1,
TimeOffset = 3.E3*I3Units.ns,
OverlapFraction = 0.75,
Discard = True)
# NOTE create the HypoFrames from all remaining SplitFrames
tray.AddModule("HypoFrameCreator", "ForceHypoFrames",
SplitName = SplitName,
HypoName = "hypoframe", # NOTE give the stream of Hypoframes a unique name
RecoMapName = SplitPulses,
MaxTimeSeparation = 3000.*I3Units.ns) # NOTE specify that frames timely spaced too far apart should not be recombined
from icecube import linefit, lilliput
# NOTE Create variables and objects that you want to base your recombination decisions on; e.g. here a simple reconstruction
tray.AddSegment( linefit.simple,'LineFit_Masked',
inputResponse = SplitPulses,
fitName = 'LineFit_Masked',
If = which_split(SplitName) | which_split('hypoframe') ) # NOTE do this only on the SplitFrames and the HypoFrames; you can use the which_split-function to select the streams to execute on
# NOTE Run all your Tester-modules that you like. No recombination will be performed at this step yet
tray.AddModule("CylinderPulsesTester", "TestHypoCylinderPulses",
SplitName = SplitName,
HypoName = "hypoframe",
RecoMapName = SplitPulses,
HypoFitName = "LineFit_Masked",
CriticalRatio = 0.7, #0.8
CylinderRadius = 150*I3Units.meter)
# NOTE Final Judgement on the recombination is passed here, by evaluation of the decisions of the Tester-modules,
# NOTE recombined ComboFrames will be inlined at the appropriate place in the stream of SplitFrames
tray.AddModule("DecisionMaker", "FinalDecision",
SplitName = SplitName,
HypoName = "hypoframe",
RecoMapName = SplitPulses,
LikeNameList = ["TestHypoCylinderPulses"], # NOTE if any of these Testers favours recombination frames will be recombined
VetoNameList = [], # NOTE if any of these modules disfavours recombination frames will not be recombined
TrueNameList = [],
Discard = True) # NOTE if any of these module favours recombination frames will be recombined regardless
# NOTE remove all HypoFrames because we do not need any longer
tray.AddModule( lambda f: f["I3EventHeader"].sub_event_stream!="hypoframe", "KillHypoFrame")
# NOTE rerun the variables and objects on the newly created ComboFrames
tray.AddSegment(linefit.simple,'LineFit_AGAIN',
inputResponse = SplitPulses,
fitName = 'LineFit_Masked',
If = which_split(SplitName) & pypick(lambda f: not f.Has("LineFit_Masked")) )
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
cfg = yaml.load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
infile = cfg['infile_pattern'].format(**cfg)
infile = infile.replace(' ', '0')
tray = I3Tray()
"""The main L1 script"""
tray.AddModule('I3Reader',
'i3 reader',
FilenameList=[cfg['gcd_pass2'], infile])
# run online filters
online_kwargs = {}
if SPLINE_TABLES:
online_kwargs.update({
'SplineRecoAmplitudeTable':
os.path.join(SPLINE_TABLES, 'InfBareMu_mie_abs_z20a10.fits'),
'SplineRecoTimingTable':
os.path.join(SPLINE_TABLES, 'InfBareMu_mie_prob_z20a10.fits'),
'hese_followup_base_GCD_filename':
cfg['gcd_pass2'],
})
if cfg['L1_pass2_run_gfu'] is not None:
online_kwargs['gfu_enabled'] = cfg['L1_pass2_run_gfu']
tray.AddSegment(OnlineFilter,
"OnlineFilter",
decode=False,
simulation=True,
vemcal_enabled=False,
**online_kwargs)
# make random service
seed = cfg['seed'] + run_number
filter_mask_randoms = phys_services.I3GSLRandomService(seed)
# override MinBias Prescale
filterconfigs = filter_globals.filter_pairs + filter_globals.sdst_pairs
print(cfg['L1_min_bias_prescale'])
if cfg['L1_min_bias_prescale']:
for i, filtertuple in enumerate(filterconfigs):
if filtertuple[0] == filter_globals.FilterMinBias:
del filterconfigs[i]
filterconfigs.append(
(filtertuple[0], cfg['L1_min_bias_prescale']))
break
print(filterconfigs)
# Generate filter Masks for all P frames
tray.AddModule(filter_tools.FilterMaskMaker,
"MakeFilterMasks",
OutputMaskName=filter_globals.filter_mask,
FilterConfigs=filterconfigs,
RandomService=filter_mask_randoms)
# Merge the FilterMasks
tray.AddModule("OrPframeFilterMasks",
"make_q_filtermask",
InputName=filter_globals.filter_mask,
OutputName=filter_globals.qfilter_mask)
# Q+P frame specific keep module needs to go first, as KeepFromSubstram
# will rename things, let's rename post keep.
def is_Q(frame):
return frame.Stop == frame.DAQ
simulation_keeps = [
'BackgroundI3MCTree', 'BackgroundI3MCTreePEcounts',
'BackgroundI3MCPESeriesMap', 'BackgroundI3MCTree_preMuonProp',
'BackgroundMMCTrackList', 'BeaconLaunches', 'CorsikaInteractionHeight',
'CorsikaWeightMap', 'EventProperties', 'GenerationSpec',
'I3LinearizedMCTree', 'I3MCTree', 'I3MCTreePEcounts',
'I3MCTree_preMuonProp', 'I3MCPESeriesMap', 'I3MCPulseSeriesMap',
'I3MCPulseSeriesMapParticleIDMap', 'I3MCWeightDict',
'LeptonInjectorProperties', 'MCHitSeriesMap', 'MCPrimary',
'MCPrimaryInfo', 'MMCTrackList', 'PolyplopiaInfo', 'PolyplopiaPrimary',
'RNGState', 'SignalI3MCPEs', 'SimTrimmer', 'TimeShift', 'WIMP_params'
] + muongun_keys
keep_before_merge = filter_globals.q_frame_keeps + [
'InIceDSTPulses', 'IceTopDSTPulses', 'CalibratedWaveformRange',
'UncleanedInIcePulsesTimeRange', 'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange', 'CalibrationErrata',
'SaturationWindows', 'InIceRawData', 'IceTopRawData'
] + simulation_keeps
tray.AddModule("Keep",
"keep_before_merge",
keys=keep_before_merge,
If=is_Q)
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheckAll",
FilterNameList=filter_globals.filter_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedAnyFilter",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def do_save_just_superdst(frame):
if frame.Has("PassedAnyFilter"):
if not frame["PassedAnyFilter"].value:
return True
else:
return False
else:
print("Failed to find key frame Bool!!")
return False
keep_only_superdsts = filter_globals.keep_nofilterpass + [
'PassedAnyFilter', 'InIceDSTPulses', 'IceTopDSTPulses',
'SplitUncleanedInIcePulses', 'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange', 'RNGState'
] + simulation_keeps
tray.AddModule("Keep",
"KeepOnlySuperDSTs",
keys=keep_only_superdsts,
If=do_save_just_superdst)
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheckSDST",
FilterNameList=filter_globals.sdst_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedKeepSuperDSTOnly",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def dont_save_superdst(frame):
if frame.Has("PassedKeepSuperDSTOnly") and \
frame.Has("PassedAnyFilter"):
if frame["PassedAnyFilter"].value:
return False
elif not frame["PassedKeepSuperDSTOnly"].value:
return True
else:
return False
else:
print("Failed to find key frame Bool!!")
return False
tray.AddModule("Keep",
"KeepOnlyDSTs",
keys=filter_globals.keep_dst_only + [
"PassedAnyFilter", "assedKeepSuperDSTOnly",
filter_globals.eventheader
] + muongun_keys,
If=dont_save_superdst)
tray.AddModule("KeepFromSubstream",
"null_stream",
StreamName=filter_globals.NullSplitter,
KeepKeys=filter_globals.null_split_keeps)
in_ice_keeps = filter_globals.inice_split_keeps + \
filter_globals.onlinel2filter_keeps
in_ice_keeps = in_ice_keeps + [
'I3EventHeader', 'SplitUncleanedInIcePulses',
'TriggerSplitterLaunchWindow', 'I3TriggerHierarchy',
'GCFilter_GCFilterMJD'
] + muongun_keys
tray.AddModule(
"Keep",
"inice_keeps",
keys=in_ice_keeps,
If=which_split(split_name=filter_globals.InIceSplitter),
)
tray.AddModule("KeepFromSubstream",
"icetop_split_stream",
StreamName=filter_globals.IceTopSplitter,
KeepKeys=filter_globals.icetop_split_keeps)
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheck",
FilterNameList=filter_globals.filters_keeping_allraw,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedConventional",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def I3RawDataCleaner(frame):
if not (('PassedConventional' in frame
and frame['PassedConventional'].value == True) or
('SimTrimmer' in frame and frame['SimTrimmer'].value == True)):
frame.Delete('InIceRawData')
frame.Delete('IceTopRawData')
tray.AddModule(I3RawDataCleaner,
"CleanErrataForConventional",
Streams=[icetray.I3Frame.DAQ])
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace(' ', '0')
tray.AddModule("I3Writer",
"EventWriter",
filename=outfile,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation
])
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(options, stats={}):
"""The main L1 script"""
tray = I3Tray()
# make list of input files from GCD and infile
'''
if isinstance(options['infile'],list):
infiles = [options['gcdfile']]
infiles.extend(options['infile'])
else:
infiles = [options['gcdfile'], options['infile']]
'''
infiles = [options['gcdfile']] + glob.glob(options['infile'])
print 'infiles: ', infiles
# test access to input and output files
for f in infiles:
if not os.access(f, os.R_OK):
raise Exception('Cannot read from %s' % f)
def test_write(f):
if f:
try:
open(f, 'w')
except IOError:
raise Exception('Cannot write to %s' % f)
finally:
os.remove(f)
test_write(options['outfile'])
########################################
tray = I3Tray()
tray.AddModule("I3Reader", "reader", filenamelist=infiles)
# Ramesh. Ignore frame where only MCPrimary exist after detector simulation.
def inframe(frame):
#print frame.keys()
#return 'IceTopRawData' in frame
return 'I3TriggerHierarchy' in frame
if options['sdstarchive']:
# do some renaming and copying to clean up
def trigger_cleanup(frame):
if 'DSTTriggers' in frame and 'I3TriggerHierarchy' not in frame:
frame[
'I3TriggerHierarchy'] = dataclasses.I3TriggerHierarchy.from_frame(
frame, 'DSTTriggers')
tray.AddModule(trigger_cleanup,
'trigger_cleanup',
Streams=[icetray.I3Frame.DAQ])
tray.Add(inframe, 'inframe',
Streams=[icetray.I3Frame.DAQ
]) #Ramesh. Ignore frame where only MCPrimary exist.
from icecube import WaveCalibrator
icetray.load("DomTools", False)
icetray.load("wavedeform", False)
icetray.load("tpx", False)
# rerun IceTop calibration
tray.AddModule("I3WaveCalibrator",
name + "_WaveCalibrator_IceTop",
Launches="IceTopRawData",
Waveforms="IceTopCalibratedWaveforms",
Errata="IceTopErrata",
WaveformRange="")
tray.AddModule('I3WaveformSplitter',
name + '_WaveformSplitter_IceTop',
Force=True,
Input='IceTopCalibratedWaveforms',
PickUnsaturatedATWD=True,
HLC_ATWD='CalibratedIceTopATWD_HLC',
HLC_FADC='CalibratedIceTopFADC_HLC',
SLC='CalibratedIceTopATWD_SLC')
tray.AddModule(
'I3TopHLCPulseExtractor',
'tpx_hlc',
Waveforms=
'CalibratedIceTopATWD_HLC', #PEPulses=filter_globals.ITRecoPulses, #Ramesh
PulseInfo='IceTopHLCPulseInfo',
VEMPulses='IceTopHLCVEMPulses',
MinimumLeadingEdgeTime=-100.0 * I3Units.ns)
tray.AddModule('I3TopSLCPulseExtractor',
'tpx_slc',
Waveforms='CalibratedIceTopATWD_SLC',
PEPulses='IceTopPulses_SLC',
VEMPulses='IceTopSLCVEMPulses')
# run online filters
online_kwargs = {}
if options['photonicsdir']:
online_kwargs.update({
'PhotonicsTabledirMu':
os.path.join(options['photonicsdir'], 'SPICE1'),
'PhotonicsDriverfileMu':
os.path.join('driverfiles', 'mu_photorec.list'),
'SplineRecoAmplitudeTable':
os.path.join(options['photonicsdir'], 'splines',
'InfBareMu_mie_abs_z20a10.fits'),
'SplineRecoTimingTable':
os.path.join(options['photonicsdir'], 'splines',
'InfBareMu_mie_prob_z20a10.fits'),
'hese_followup_base_GCD_filename':
options['gcdfile'],
})
if options['sdstarchive']:
online_kwargs['sdstarchive'] = True
if options['GFU'] is not None:
online_kwargs['gfu_enabled'] = options['GFU']
online_kwargs['ehealert_followup'] = False #Ramesh
online_kwargs['hese_followup'] = False #Ramesh
# All Filter check is done here.
tray.AddSegment(OnlineFilter,
"OnlineFilter",
decode=False,
simulation=True,
vemcal_enabled=False,
**online_kwargs)
# make random service
seed = os.getpid()
filter_mask_randoms = phys_services.I3GSLRandomService(seed)
# override MinBias Prescale
filterconfigs = filter_globals.filter_pairs + filter_globals.sdst_pairs
if options['MinBiasPrescale']:
for i, filtertuple in enumerate(filterconfigs):
if filtertuple[0] == filter_globals.FilterMinBias:
del filterconfigs[i]
filterconfigs.append(
(filtertuple[0], options['MinBiasPrescale']))
break
print filterconfigs
# Generate filter Masks for all P frames
tray.AddModule(filter_tools.FilterMaskMaker,
"MakeFilterMasks",
OutputMaskName=filter_globals.filter_mask,
FilterConfigs=filterconfigs,
RandomService=filter_mask_randoms)
# Merge the FilterMasks
tray.AddModule("OrPframeFilterMasks",
"make_q_filtermask",
InputName=filter_globals.filter_mask,
OutputName=filter_globals.qfilter_mask)
#Q+P frame specific keep module needs to go first, as KeepFromSubstram
#will rename things, let's rename post keep.
def is_Q(frame):
return frame.Stop == frame.DAQ
keep_before_merge = filter_globals.q_frame_keeps + [
'InIceDSTPulses', # keep DST pulse masks
'IceTopDSTPulses',
'CalibratedWaveformRange', # keep calibration info
'UncleanedInIcePulsesTimeRange',
'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange',
'CalibrationErrata',
'SaturationWindows',
'InIceRawData', # keep raw data for now
'IceTopRawData',
'CorsikaWeightMap', # sim keys
'I3MCWeightDict',
'MCHitSeriesMap',
'MMCTrackList',
'I3MCTree',
'I3LinearizedMCTree',
'MCPrimary',
'MCPrimaryInfo',
'TimeShift', # the time shift amount
'WIMP_params', # Wimp-sim
'SimTrimmer', # for SimTrimmer flag
'I3MCPESeriesMap',
'I3MCPulseSeriesMap',
'I3MCPulseSeriesMapParticleIDMap',
]
if options['sdstarchive']:
keep_before_merge.extend(['I3SuperDSTTimeRange'])
tray.AddModule("Keep",
"keep_before_merge",
keys=keep_before_merge,
If=is_Q)
## second set of prekeeps, conditional on filter content, based on newly created Qfiltermask
## Determine if we should apply harsh keep for events that failed to pass any filter
## Note: excluding the sdst_streams entries
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheckAll",
FilterNameList=filter_globals.filter_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedAnyFilter",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def do_save_just_superdst(frame):
if frame.Has("PassedAnyFilter"):
if not frame["PassedAnyFilter"].value:
return True # <- Event failed to pass any filter.
else:
return False # <- Event passed some filter
else:
print "Failed to find key frame Bool!!"
return False
keep_only_superdsts = filter_globals.keep_nofilterpass + [
'PassedAnyFilter',
'InIceDSTPulses',
'IceTopDSTPulses',
'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange',
'CorsikaWeightMap', # sim keys
'I3MCWeightDict',
'MCHitSeriesMap',
'MMCTrackList',
'I3MCTree',
'I3LinearizedMCTree',
'MCPrimary',
'MCPrimaryInfo',
'TimeShift', # the time shift amount
'WIMP_params', # Wimp-sim
'I3MCPESeriesMap',
'I3MCPulseSeriesMap',
'I3MCPulseSeriesMapParticleIDMap',
]
if options['sdstarchive']:
keep_only_superdsts.extend([
'I3SuperDSTTimeRange',
])
tray.AddModule("Keep",
"KeepOnlySuperDSTs",
keys=keep_only_superdsts,
If=do_save_just_superdst)
## Now clean up the events that not even the SuperDST filters passed on.
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheckSDST",
FilterNameList=filter_globals.sdst_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedKeepSuperDSTOnly",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
if not options['sdstarchive']:
def dont_save_superdst(frame):
if frame.Has("PassedKeepSuperDSTOnly") and frame.Has(
"PassedAnyFilter"):
if frame["PassedAnyFilter"].value:
return False # <- these passed a regular filter, keeper
elif not frame["PassedKeepSuperDSTOnly"].value:
return True # <- Event failed to pass SDST filter.
else:
return False # <- Event passed some SDST filter
else:
print "Failed to find key frame Bool!!"
return False
tray.AddModule("Keep",
"KeepOnlyDSTs",
keys=filter_globals.keep_dst_only + [
"PassedAnyFilter", "PassedKeepSuperDSTOnly",
filter_globals.eventheader
],
If=dont_save_superdst)
## Frames should now contain only what is needed. now flatten, write/send to server
## Squish P frames back to single Q frame, one for each split:
tray.AddModule(
"KeepFromSubstream",
"null_stream",
StreamName=filter_globals.NullSplitter,
KeepKeys=filter_globals.null_split_keeps,
)
in_ice_keeps = filter_globals.inice_split_keeps + filter_globals.onlinel2filter_keeps
in_ice_keeps = in_ice_keeps + [
'I3EventHeader', 'SplitUncleanedInIcePulses',
'TriggerSplitterLaunchWindow', 'I3TriggerHierarchy',
'GCFilter_GCFilterMJD'
]
tray.AddModule(
"Keep",
"inice_keeps",
keys=in_ice_keeps,
If=which_split(split_name=filter_globals.InIceSplitter),
)
tray.AddModule(
"KeepFromSubstream",
"icetop_split_stream",
StreamName=filter_globals.IceTopSplitter,
KeepKeys=filter_globals.icetop_split_keeps,
)
# Apply small keep list (SuperDST/SmallTrig/DST/FilterMask for non-filter passers
# Remove I3DAQData object for events not passing one of the 'filters_keeping_allraw'
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheck",
FilterNameList=filter_globals.filters_keeping_allraw,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedConventional",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
## Clean out the Raw Data when not passing conventional filter
def I3RawDataCleaner(frame):
if not (('PassedConventional' in frame
and frame['PassedConventional'].value == True) or
('SimTrimmer' in frame and frame['SimTrimmer'].value == True)):
frame.Delete('InIceRawData')
frame.Delete('IceTopRawData')
tray.AddModule(I3RawDataCleaner,
"CleanErrataForConventional",
Streams=[icetray.I3Frame.DAQ])
###################################################################
########### WRITE STUFF ##########################
###################################################################
# clean up special case for bz2 files to get around empty file bug
bzip2_files = []
if options['outfile'] and options['outfile'].endswith('.bz2'):
options['outfile'] = options['outfile'][:-4]
bzip2_files.append(options['outfile'])
# Write the physics and DAQ frames
tray.AddModule("I3Writer",
"EventWriter",
filename=options['outfile'],
Streams=[icetray.I3Frame.DAQ, icetray.I3Frame.Physics])
tray.AddModule("TrashCan", "byebye")
# make it go
if options['num'] >= 0:
tray.Execute(options['num'])
else:
tray.Execute()
tray.Finish()
# print more CPU usage info. than speicifed by default
tray.PrintUsage(fraction=1.0)
for entry in tray.Usage():
stats[entry.key()] = entry.data().usertime
if bzip2_files:
# now do bzip2
subprocess.check_call(['/usr/bin/bzip2', '-f'] + bzip2_files)
# clean up forcefully in case we're running this in a loop
del tray
tray.AddModule("KeepFromSubstream","null_stream",
StreamName = filter_globals.NullSplitter,
KeepKeys = filter_globals.null_split_keeps,
)
from icecube.phys_services.which_split import which_split
in_ice_keeps = filter_globals.inice_split_keeps + filter_globals.onlinel2filter_keeps
in_ice_keeps = in_ice_keeps + ['I3EventHeader',
'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'TriggerSplitterLaunchWindow',
'I3TriggerHierarchy',
'GCFilter_GCFilterMJD'] + gcd_keeps
tray.AddModule("Keep", "inice_keeps",
keys = in_ice_keeps,
If = which_split(split_name=filter_globals.InIceSplitter),
)
tray.AddModule("KeepFromSubstream","icetop_split_stream",
StreamName = filter_globals.IceTopSplitter,
KeepKeys = filter_globals.icetop_split_keeps,
)
# Apply small keep list (SuperDST/SmallTrig/DST/FilterMask for non-filter passers
# Remove I3DAQData object for events not passing one of the 'filters_keeping_allraw'
tray.AddModule("I3IcePickModule<FilterMaskFilter>","filterMaskCheck",
FilterNameList = filter_globals.filters_keeping_allraw,
FilterResultName = filter_globals.qfilter_mask,
DecisionName = "PassedConventional",
DiscardEvents = False,
def main(options, stats={}):
"""The main L1 script"""
########################################
gcdfile = options['gcdfile']
infile = options['infile']
outfile = options['outfile']
move = str(options['move'])
print(move)
osg = options['osg']
if osg == 'True':
def copy_to_OSG(NPX_file):
subprocess.check_call([
'globus-url-copy', '-nodcau', '-rst', NPX_file,
"file:" + os.getcwd() + '/' + str(NPX_file.split('/')[-1])
])
print('copy worked')
def copy_to_NPX(NPX_file):
subprocess.check_call([
'globus-url-copy', '-nodcau', '-rst',
"file:" + os.getcwd() + '/' + str(NPX_file.split('/')[-1]),
NPX_file
])
gcdfile_NPX = str('gsiftp://gridftp.icecube.wisc.edu' + gcdfile)
gcdfile = str(os.getcwd() + '/' + gcdfile.split('/')[-1])
infile_NPX = str('gsiftp://gridftp.icecube.wisc.edu' + infile)
infile = str(os.getcwd() + '/' + infile.split('/')[-1])
copy_to_OSG(gcdfile_NPX)
copy_to_OSG(infile_NPX)
outfile = str('gsiftp://gridftp.icecube.wisc.edu' + outfile)
outfile_temp = str(os.getcwd() + '/' + outfile.split('/')[-1])
'''
try:
copy_to_OSG(outfile)
except:
print('Outfile does not exsist. Continuing...')
if os.path.isfile(outfile_temp):
print('Outfile already processed. Exiting.')
sys.exit()
'''
#outfile_temp = str(os.getcwd()+'/'+outfile.split('/')[-1])
else:
outfile_temp = '/data/ana/SterileNeutrino/IC86/HighEnergy/MC/scripts/temp/' + outfile.split(
'/')[-1]
outfile = outfile
infiles = [gcdfile, infile]
print(infiles)
tray = I3Tray()
tray.Add(dataio.I3Reader, "reader", FilenameList=infiles)
# run online filters
online_kwargs = {}
if options['photonicsdir']:
online_kwargs.update({
'SplineRecoAmplitudeTable':
os.path.join(options['photonicsdir'], 'splines',
'InfBareMu_mie_abs_z20a10.fits'),
'SplineRecoTimingTable':
os.path.join(options['photonicsdir'], 'splines',
'InfBareMu_mie_prob_z20a10.fits'),
#'hese_followup_base_GCD_filename': gcdfile,
'alert_followup_base_GCD_filename':
gcdfile, #base_GCD_filename
})
if options['GFU'] is not None:
online_kwargs['gfu_enabled'] = options['GFU']
tray.AddSegment(
OnlineFilter,
"OnlineFilter",
decode=False,
simulation=True,
vemcal_enabled=False,
#base_GCD_filename = gcdfile,
#alert_followup_omit_GCD_diff = True,
**online_kwargs)
# make random service
seed = os.getpid()
filter_mask_randoms = phys_services.I3GSLRandomService(seed)
# override MinBias Prescale
filterconfigs = filter_globals.filter_pairs + filter_globals.sdst_pairs
if options['MinBiasPrescale']:
for i, filtertuple in enumerate(filterconfigs):
if filtertuple[0] == filter_globals.FilterMinBias:
del filterconfigs[i]
filterconfigs.append(
(filtertuple[0], options['MinBiasPrescale']))
break
print(filterconfigs)
# Generate filter Masks for all P frames
tray.AddModule(filter_tools.FilterMaskMaker,
"MakeFilterMasks",
OutputMaskName=filter_globals.filter_mask,
FilterConfigs=filterconfigs,
RandomService=filter_mask_randoms)
# Merge the FilterMasks
tray.AddModule("OrPframeFilterMasks",
"make_q_filtermask",
InputName=filter_globals.filter_mask,
OutputName=filter_globals.qfilter_mask)
#Q+P frame specific keep module needs to go first, as KeepFromSubstram
#will rename things, let's rename post keep.
def is_Q(frame):
return frame.Stop == frame.DAQ
simulation_keeps = [
'BackgroundI3MCTree',
'BackgroundI3MCTreePEcounts',
'BackgroundI3MCPESeriesMap',
'BackgroundI3MCTree_preMuonProp',
'BackgroundMMCTrackList',
'BeaconLaunches',
'CorsikaInteractionHeight',
'CorsikaWeightMap',
'EventProperties',
'GenerationSpec',
'I3LinearizedMCTree',
'I3MCTree',
'I3MCTreePEcounts',
'I3MCTree_preMuonProp',
'I3MCPESeriesMap',
'I3MCPulseSeriesMap',
'I3MCPulseSeriesMapParticleIDMap',
'I3MCWeightDict',
'IceXModelNumber', #Modified May 16th.
'MultisimModes', #Modified May 16th.
'MultisimPhases', #Modified May 16th.
'MultisimAmplitudes', #Modified May 16th.
'LeptonInjectorProperties',
'MCHitSeriesMap',
'MCPrimary',
'MCPrimaryInfo',
'MMCTrackList',
'PolyplopiaInfo',
'PolyplopiaPrimary',
'RNGState',
'SignalI3MCPEs',
'SimTrimmer', # for SimTrimmer flag
'TimeShift', # the time shift amount
'WIMP_params', # Wimp-sim
]
keep_before_merge = filter_globals.q_frame_keeps + [
'InIceDSTPulses', # keep DST pulse masks
'IceTopDSTPulses',
'LeptonInjectorProperties',
'CalibratedWaveformRange', # keep calibration info
'UncleanedInIcePulsesTimeRange',
'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange',
'CalibrationErrata',
'SaturationWindows',
'InIceRawData', # keep raw data for now
'IceTopRawData',
] + simulation_keeps
tray.AddModule("Keep",
"keep_before_merge",
keys=keep_before_merge,
If=is_Q)
## second set of prekeeps, conditional on filter content, based on newly created Qfiltermask
#Determine if we should apply harsh keep for events that failed to pass any filter
## Note: excluding the sdst_streams entries
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheckAll",
FilterNameList=filter_globals.filter_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedAnyFilter",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def do_save_just_superdst(frame):
if frame.Has("PassedAnyFilter"):
if not frame["PassedAnyFilter"].value:
return True # <- Event failed to pass any filter.
else:
return False # <- Event passed some filter
else:
print("Failed to find key frame Bool!!")
return False
keep_only_superdsts = filter_globals.keep_nofilterpass + [
'PassedAnyFilter',
'LeptonInjectorProperties',
'InIceDSTPulses',
'IceTopDSTPulses',
'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange',
'RNGState',
] + simulation_keeps
tray.AddModule("Keep",
"KeepOnlySuperDSTs",
keys=keep_only_superdsts,
If=do_save_just_superdst)
## Now clean up the events that not even the SuperDST filters passed on.
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheckSDST",
FilterNameList=filter_globals.sdst_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedKeepSuperDSTOnly",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def dont_save_superdst(frame):
if frame.Has("PassedKeepSuperDSTOnly") and frame.Has(
"PassedAnyFilter"):
if frame["PassedAnyFilter"].value:
return False # <- these passed a regular filter, keeper
elif not frame["PassedKeepSuperDSTOnly"].value:
return True # <- Event failed to pass SDST filter.
else:
return False # <- Event passed some SDST filter
else:
print("Failed to find key frame Bool!!")
return False
tray.AddModule("Keep",
"KeepOnlyDSTs",
keys=filter_globals.keep_dst_only + [
"PassedAnyFilter", "PassedKeepSuperDSTOnly",
filter_globals.eventheader
] + simulation_keeps,
If=dont_save_superdst)
tray.Add('Dump')
## Frames should now contain only what is needed. now flatten, write/send to server
## Squish P frames back to single Q frame, one for each split:
tray.AddModule(
"KeepFromSubstream",
"null_stream",
StreamName=filter_globals.NullSplitter,
KeepKeys=filter_globals.null_split_keeps,
)
in_ice_keeps = filter_globals.inice_split_keeps + filter_globals.onlinel2filter_keeps
in_ice_keeps = in_ice_keeps + [
'I3EventHeader',
'LeptonInjectorProperties',
'SplitUncleanedInIcePulses',
'TriggerSplitterLaunchWindow',
'I3TriggerHierarchy',
'SplitUncleanedInIceDSTPulsesTimeRange', #Added by Kotoyo
'GCFilter_GCFilterMJD'
]
tray.AddModule(
"Keep",
"inice_keeps",
keys=in_ice_keeps,
If=which_split(split_name=filter_globals.InIceSplitter),
)
tray.AddModule(
"KeepFromSubstream",
"icetop_split_stream",
StreamName=filter_globals.IceTopSplitter,
KeepKeys=filter_globals.icetop_split_keeps,
)
# Apply small keep list (SuperDST/SmallTrig/DST/FilterMask for non-filter passers
# Remove I3DAQData object for events not passing one of the 'filters_keeping_allraw'
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheck",
FilterNameList=filter_globals.filters_keeping_allraw,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedConventional",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
## Clean out the Raw Data when not passing conventional filter
def I3RawDataCleaner(frame):
if not (('PassedConventional' in frame
and frame['PassedConventional'].value == True) or
('SimTrimmer' in frame and frame['SimTrimmer'].value == True)):
frame.Delete('InIceRawData')
frame.Delete('IceTopRawData')
tray.AddModule(I3RawDataCleaner,
"CleanErrataForConventional",
Streams=[icetray.I3Frame.DAQ])
###################################################################
########### WRITE STUFF ##########################
###################################################################
# clean up special case for bz2 files to get around empty file bug
'''
bzip2_files = []
if options['outfile'] and options['outfile'].endswith('.bz2'):
options['outfile'] = options['outfile'][:-4]
bzip2_files.append(options['outfile'])
'''
# Write the physics and DAQ frames
i3streams = [
icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.TrayInfo,
icetray.I3Frame.Simulation
]
if move == "True":
tray.AddModule("I3Writer", "i3writer")(("Filename", outfile_temp),
("Streams", i3streams))
else:
tray.AddModule("I3Writer", "i3writer")(("Filename", outfile),
("Streams", i3streams))
'''
tray.AddModule("I3Writer", "EventWriter",
filename=options['outfile'],
Streams=[icetray.I3Frame.DAQ,icetray.I3Frame.Physics,
icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation]
)
'''
tray.AddModule("TrashCan", "byebye")
if options['num'] >= 0:
tray.Execute(options['num'])
else:
tray.Execute()
tray.Finish()
# print more CPU usage info. than speicifed by default
tray.PrintUsage(fraction=1.0)
for entry in tray.Usage():
stats[entry.key()] = entry.data().usertime
'''
if bzip2_files:
# now do bzip2
if os.path.exists('/usr/bin/bzip2'):
subprocess.check_call(['/usr/bin/bzip2', '-f']+bzip2_files)
elif os.path.exists('/bin/bzip2'):
subprocess.check_call(['/bin/bzip2', '-f']+bzip2_files)
else:
raise Exception('Cannot find bzip2')
'''
# clean up forcefully in case we're running this in a loop
if move == 'True':
if osg == "True":
copy_to_NPX(outfile)
else:
os.system(str("mv " + str(outfile_temp) + " " + str(outfile)))
else:
if osg == "True":
print('You need move option to be True if using the OSG')
pass
del tray
def Split_and_Recombine(
tray,
name,
LineFitName="LineFit",
SplitName="IceHiveSplit",
OriginalPulses="InIcePulses",
SplitPulses="SplitInIcePulses",
CleanedSplitPulsesName="CleanSplitInIcePulses",
DesignatedPulsesName="RecoInIcePulses",
TriggerHierarchyName="QTriggerHierarchy",
InIceTriggerIDs=[1006, 1007, 1011,
21001], #[(SMT8),(string),(SMT3),(volume)]
AddTimeWindow=True,
Clean=True,
LineFitReco=True,
Recombine=True,
Recover=False,
):
"""
A tray-segment to perform event-splitting and event-recombination at low-level in data processing
:param DesignatedPulsesName: The end product pulses will be written to this key
:param LineFitName: The end produce LineFit will be written to this key
:param OriginalPulses: Name of the original pulse in the Q-frame that will be split
:param SplitName: Name of the splitter and therefore the subevent-stream that is delivered
:param SplitPulses: Name of the SplitPulses in the P-frame
:param TriggerHierarchyName: Name of the TriggerHierarchy in the Q-frame
:param LineFitName: Name of that the LineFit reconstruction will be given
:param AddTimeWindow: Set to True if a time window should be added (needed for EHE filter, I3TriggerSplitter does this too)
:param CleanedSplitPulsesName: Name of the Cleaned Pulses if option 'Clean' is enabled
:param Clean: Choose if a round of HiveCleaning (similar to RT-Cleaning) should commence on the output-pulses; improves reconstructability
:param LineFitReco: make a importvedLineFit reconstruction; needed for Recombine and Recover option
:param Recombine: recombine wrongly split events with CoincSuite
:param Recover: try to recover lost pulses, by tracing back the track
The Output are Q-frames containing:
-[SplitNameSplitCount]: The number of created P-frames (SplitFrames) by the splitter
-[SplitNameReducedCount]: The number of removed splits by means of recombinations; [SplitNameSplitCount]-[SplitNameReducedCount] is the effective remaining number of SplitFrames
-[SplitPulses_Physics]: All clustered Pulses in the OriginalPulseSeries; identical to the union of all SplitPulses in the SubFrames
-[SplitPulses_Noise]: All not clustered Pulses in the OriginalPulseSeries; identical to the inversion of SplitPulses_Physics
and P-frames containing:
-[SplitPulses]: The split PulseSeriesMapMask, which contains the clustered Pulses as selected by the Splitter
-[HCSplitPulses]: The split PulseSeriesMapMask after the application of HiveCleaning, if the 'Clean'-option has been chosen
-[SplitPulsesTimeRange]: The time range of the PulseSeriesMapMask equal to [first-pulse.time, last-pulse.time]
-[SplitPulses_Noised]: The SplitPulses plus all noise in the feasible time-range; might be helpful for native reconstructions which do not depend on hit/pulse-cleaning or which explicitly require noised pulses, because they assume a noise-probability
-[I3TriggerHierarchy]: The SubTrigger Hierarchy containing only Triggers participating in this event
-[LineFitName]: A LineFit reconstruction
"""
from icecube import icetray, dataclasses, phys_services, IceHive
from I3Tray import I3Units
from icecube.icetray import pypick
from icecube.phys_services.which_split import which_split
if LineFitReco:
try:
from icecube import linefit
except:
raise RuntimeError(
"for the option LineFitReco you need to have LineFit installed"
)
if Recombine:
try:
from icecube import CoincSuite
except:
raise RuntimeError(
"for the options 'Recombine' you need to have CoincSuite installed"
)
if AddTimeWindow:
try:
from icecube import CoincSuite
except:
raise RuntimeError(
"for the options 'AddTimeWindow' you need to have CoincSuite installed"
)
if Recover:
try:
from icecube import PulseRecover
except:
raise RuntimeError(
"for the option 'Clean' you need to have PulseRecover installed.\nYou can fin it in SVN/sandbox/mzoll/PulseRecover/branches/LineFit."
)
#=====================
# IceHive Event Splitting
#======================
#Run the HiveSplitter: create P-frames containing its splits
from icecube import IceHive
singleRings = IceHive.RingLimits()
singleRings.AddLimitPair(IceHive.LimitPair(-255., 255.))
singleRings.AddLimitPair(IceHive.LimitPair(-272.7, 272.7))
singleRings.AddLimitPair(IceHive.LimitPair(-165.8, 165.8))
doubleRings = IceHive.RingLimits()
doubleRings.AddLimitPair(IceHive.LimitPair(-130., 70.))
doubleRings.AddLimitPair(IceHive.LimitPair(-131.5, 131.5))
doubleRings.AddLimitPair(IceHive.LimitPair(-40.8, 40.8))
tripleRings = IceHive.RingLimits()
tripleRings.AddLimitPair(IceHive.LimitPair(-130., 70.))
tripleRings.AddLimitPair(IceHive.LimitPair(-144.1, 144.1))
tripleRings.AddLimitPair(IceHive.LimitPair(-124.7, 124.7))
tripleRings.AddLimitPair(IceHive.LimitPair(-82.8, 82.8))
singleVicinity = singleRings
doubleVicinity = doubleRings
tripleVicinity = tripleRings
#NOTE FUTURE a more stringend set of limits
#singleVicinity = IceHive.RingLimits()
#singleVicinity.AddLimitPair(IceHive.LimitPair(-100., 100.))
#singleVicinity.AddLimitPair(IceHive.LimitPair(-100., 100.))
#doubleVicinity = IceHive.RingLimits()
#doubleVicinity.AddLimitPair(IceHive.LimitPair(-100., 100.))
#doubleVicinity.AddLimitPair(IceHive.LimitPair(-100., 100.))
#doubleVicinity.AddLimitPair(IceHive.LimitPair(-100., 100.))
#tripleVicinity = IceHive.RingLimits()
#tripleVicinity.AddLimitPair(IceHive.LimitPair(-100., 100.))
#tripleVicinity.AddLimitPair(IceHive.LimitPair(-100., 100.))
#tripleVicinity.AddLimitPair(IceHive.LimitPair(-100., 100.))
#tripleVicinity.AddLimitPair(IceHive.LimitPair(-100., 100.))
tray.AddModule(
"I3IceHive<I3RecoPulse>",
SplitName,
InputName=OriginalPulses,
OutputName=SplitPulses,
Multiplicity=3,
TimeWindow=2000. * I3Units.ns,
TimeCVMinus=300. * I3Units.ns,
TimeCVPlus=300. * I3Units.ns,
TimeCNMinus=200. * I3Units.ns,
TimeCNPlus=200. * I3Units.ns,
TimeStatic=200. * I3Units.ns,
SingleDenseRingLimits=singleRings,
DoubleDenseRingLimits=doubleRings,
TripleDenseRingLimits=tripleRings,
SingleDenseRingVicinity=singleVicinity,
DoubleDenseRingVicinity=doubleVicinity,
TripleDenseRingVicinity=tripleVicinity,
SaveSplitCount=True,
UpdateTriggerHierarchy=True,
TrigHierName=TriggerHierarchyName,
#TriggerConfigIDs= InIceTriggerIDs,
#NoSplitDt = 10000,
ReadoutWindowMinus=4000. * I3Units.ns,
ReadoutWindowPlus=6000. * I3Units.ns)
RecoPulses = SplitPulses #NOTE Make an alias on which pulses Reconstructions should commence
if (Clean):
cleanVicinity = IceHive.RingLimits()
cleanVicinity.AddLimitPair(IceHive.LimitPair(-70., 70.))
cleanVicinity.AddLimitPair(IceHive.LimitPair(-70., 70.))
tray.AddModule(
"I3HiveCleaning<I3RecoPulse>",
"HiveClean",
InputName=SplitPulses,
OutputName=CleanedSplitPulsesName,
TimeStaticMinus=600. * I3Units.ns, #NOTE default is 200.
TimeStaticPlus=600. * I3Units.ns, #NOTE default is 200.
SingleDenseRingVicinity=cleanVicinity,
DoubleDenseRingVicinity=cleanVicinity,
TripleDenseRingVicinity=cleanVicinity,
If=which_split(SplitName))
RecoPulses = CleanedSplitPulsesName
if Recombine:
tray.AddModule(
lambda f: f.Put(SplitName + "ReducedCount", icetray.I3Int(0)),
"ReducedCountMaker",
Streams=[icetray.I3Frame.DAQ])
#=========================
# CoincSuite Recombinations
#=========================
if (Clean and Recombine):
tray.AddModule(CoincSuite.discardEmptySplits,
"removeEmptySplit",
SplitName=SplitName,
PulsesName=RecoPulses)
if Recombine:
tray.AddModule("AfterpulseDiscard",
"AfterpulseDiscard",
SplitName=SplitName,
RecoMapName=SplitPulses,
QTotFraction=.1,
TimeOffset=3000. * I3Units.ns,
OverlapFraction=0.5,
Discard=True)
tray.AddModule("HypoFrameCreator",
"HypoFrameCreator",
SplitName=SplitName,
HypoName="hypoframe",
RecoMapName=RecoPulses,
MaxTimeSeparation=3000. * I3Units.ns)
### LineFit, a fast and simple reconstruction
tray.AddSegment(linefit.simple,
'LineFit',
inputResponse=RecoPulses,
fitName=LineFitName,
If=(which_split(split_name=SplitName)
| which_split(split_name='hypoframe')))
tray.AddModule(
"TrackSystemTester",
"TestHypoTrackSystem",
SplitName=SplitName,
HypoName="hypoframe",
RecoMapName=SplitPulses,
HypoFitName=LineFitName,
CriticalRatio=0.7, #0.8
CylinderRadius=150. * I3Units.meter,
ResTimeWindow=dataclasses.make_pair(
-float("inf"), float("inf")
), #FUTURE dataclasses.make_pair(-200*I3Units.ns,200*I3Units.ns),
ParticleSpeed=float("nan")) #FUTURE dataclasses.I3Constants.c,
tray.AddModule(
"TrackSystemTester",
"TestMutualTrackSystem",
SplitName=SplitName,
HypoName="hypoframe",
RecoMapName=SplitPulses,
RecoFitName=LineFitName,
CriticalRatio=0.7, #0.8
CylinderRadius=150. * I3Units.meter,
ResTimeWindow=dataclasses.make_pair(
-float("inf"), float("inf")
), #FUTURE dataclasses.make_pair(-200*I3Units.ns,200*I3Units.ns),
ParticleSpeed=float("nan"), #FUTURE dataclasses.I3Constants.c,
MutualCompare=True)
tray.AddModule("AlignmentTester",
"TestHypoAlignment",
SplitName=SplitName,
HypoName="hypoframe",
HypoFitName=LineFitName,
RecoFitName=LineFitName,
CriticalAngle=25. * I3Units.degree,
CriticalDistance=20. * I3Units.meter)
tray.AddModule("AlignmentTester",
"TestMutualAlignment",
SplitName=SplitName,
HypoName="hypoframe",
HypoFitName=LineFitName,
RecoFitName=LineFitName,
CriticalAngle=25. * I3Units.degree,
CriticalDistance=20. * I3Units.meter,
MutualCompare=True)
tray.AddModule("SpeedTester",
"TestSpeed",
SplitName=SplitName,
HypoName="hypoframe",
HypoFitName=LineFitName,
SpeedUpperCut=0.35 * I3Units.m / I3Units.ns,
SpeedLowerCut=0.15 * I3Units.m / I3Units.ns)
tray.AddModule("cogCausalConnectTester",
"TestcogCausalConnect",
SplitName=SplitName,
HypoName="hypoframe",
RecoMapName=RecoPulses,
HypoFitName=LineFitName,
TravelTimeResidual=dataclasses.make_pair(
-1000. * I3Units.ns, 1000. * I3Units.ns),
WallTime=3000. * I3Units.ns,
MaxVerticalDist=700. * I3Units.m,
MaxHorizontalDist=700. * I3Units.m)
#where recombinations happen, for real
RecombineKeys = [SplitPulses + "_Noised"]
if (Clean):
RecombineKeys.append(CleanedSplitPulsesName)
tray.AddModule("DecisionMaker",
"FinalDecision",
SplitName=SplitName,
HypoName="hypoframe",
RecoMapName=SplitPulses,
LikeNameList=[
"TestHypoTrackSystem", "TestMutualTrackSystem",
"TestHypoAlignment", "TestMutualAlignment"
],
VetoNameList=["TestcogCausalConnect", "TestSpeed"],
TrueNameList=[],
RecombineRecoMaps=RecombineKeys,
Discard=True)
#discard the hypoframes, as they are of no use anymore
tray.AddModule(
lambda f: f['I3EventHeader'].sub_event_stream != "hypoframe",
"KillHypoFrame")
#=================
# REPEAT and uniformize
#=================
#Here procedures need to be repeated, which have not yet been performed on the recombined frames
# if (Clean):
# from icecube import IceHive
# tray.AddModule("I3HiveCleaning<I3RecoPulse>","HiveClean_AGAIN",
# InputName = SplitPulses,
# OutputName = CleanedSplitPulsesName,
# TimeStaticMinus=600.*I3Units.ns,#NOTE default is 200.
# TimeStaticPlus=600.*I3Units.ns, #NOTE default is 200.
# SingleDenseRingVicinity=cleanVicinity,
# DoubleDenseRingVicinity=cleanVicinity,
# TripleDenseRingVicinity=cleanVicinity,
# If = which_split(SplitName) & pypick(lambda f: not f.Has(CleanedSplitPulsesName)))
if (AddTimeWindow):
tray.AddModule(CoincSuite.createTimeWindow,
"HiveTimeWindow",
InputPulses=SplitPulses,
Output="TriggerSplitterLaunchWindow",
If=which_split(SplitName))
#NOTE Drop-in fix for TriggerHierarchies, which need to be present in the frame for certain filters
def ClipTriggerHierarchy(frame):
""" if frames do not have I3TriggerHierarchy put it there by clipping in time"""
qth = frame[TriggerHierarchyName]
tw = frame["TriggerSplitterLaunchWindow"]
th = IceHive.clip_TriggerHierarchy(
qth, tw,
[1011, 1006, 1007, 21001]) #SMT8, SMT3, String, Volume-trigger
frame.Put("I3TriggerHierarchy", th)
tray.AddModule(ClipTriggerHierarchy,
"ClipTriggers",
If=which_split(SplitName)
& pypick(lambda f: not f.Has("I3TriggerHierarchy")))
if Recombine:
tray.AddSegment(linefit.simple,
'LineFit_AGAIN',
inputResponse=RecoPulses,
fitName=LineFitName,
If=which_split(SplitName)
& pypick(lambda f: not f.Has(LineFitName)))
### copy the key at 'filter_globals.SplitRTCleanedInIcePulses' to 'pulses'
tray.AddModule("Copy",
"copy_AGAIN",
Keys=[RecoPulses, DesignatedPulsesName],
If=which_split(SplitName))
###
# option to recover crutial pulses, which have been lost
###
if (Recover):
tray.AddModule(
"I3SimplePulseRecover",
"SimplePulseRecover", #TODO options switch to IterativePulseRecover here!
PhysicsPulsesName=RecoPulses,
TrackName=LineFitName,
DiscoverablePulsesName=SplitPulses + "_Noised",
OutputPulsesName=RecoPulses + "Plus",
OutputTrackName=LineFitName + "Plus",
TimeResidualWindow=dataclasses.make_pair(-500. * I3Units.ns,
+500. * I3Units.ns),
MaxDistance=150. * I3Units.m,
ApproxTrack=True,
TrimTrack=True,
BackwardSearch=True,
ForwardSearch=False,
Deadtime=0.0,
If=which_split(SplitName))
def pick_recover():
def f(frame):
if (frame.Stop == icetray.I3Frame.Physics):
return (frame[RecoPulses + 'Plus' + "_discNCh"].value >= 2
and
frame[RecoPulses + 'Plus' + "_discQ"].value >= 1.5)
return False
return icetray.pypick(f)
tray.AddModule(
"Delete",
"delete_TRICE",
Keys=[DesignatedPulsesName, LineFitName, LineFitName + "Params"],
If=which_split(SplitName) & pick_recover())
tray.AddModule("Copy",
"copy_TRICE",
Keys=[RecoPulses + 'Plus', DesignatedPulsesName],
If=which_split(SplitName) & pick_recover())
tray.AddSegment(linefit.simple,
'LineFit_TRICE',
inputResponse=DesignatedPulsesName,
fitName=LineFitName,
If=which_split(SplitName) & pick_recover())
nch=frame['NCh'] # post-I3PODHolder (SVN commit 1914)
#nch=frame['NCh'].value # pre-I3PODHolder (SVN commit 1914)
if nch >= nchmin:
return True
return False
return icetray.pypick(f)
tray=I3Tray()
tray.Add(split_test_frame_generator,"bogogenesis")
#tray.Add("Dump","truck")
tray.Add(schizzoparanoia,"null",
ShouldRunOn=set([('null',0)]),
If=which_split('null'),
)
tray.Add(schizzoparanoia,"foo_bar",
ShouldRunOn=set([('foo',i) for i in range(Nfoo)]+[('bar',j) for j in range(Nbar)]),
If=which_split('foo') | which_split('bar'),
)
tray.Add(schizzoparanoia,"also_foo_bar",
ShouldRunOn=set([('foo',i) for i in range(Nfoo)]+[('bar',j) for j in range(Nbar)]),
If=which_split('foo') ^ which_split('bar'),
)
tray.Add(schizzoparanoia,"foo_bar_and_Q",
ShouldRunOn=set([('',0)]+[('foo',i) for i in range(Nfoo)]+[('bar',j) for j in range(Nbar)]),
If=~which_split('null'),
def main(options, stats={}):
"""The main L1 script"""
tray = I3Tray()
if not options['infile']:
raise Exception(
'You need to specify an input file with the -i or --input parameter.'
)
if not options['gcdfile']:
raise Exception(
'You need to specify a GCD file with the -g or --gcd parameter.')
if not options['outfile']:
raise Exception(
'You need to specify an output file with the -o or --output parameter.'
)
log_levels = {
"error": icetray.I3LogLevel.LOG_ERROR,
"warn": icetray.I3LogLevel.LOG_WARN,
"info": icetray.I3LogLevel.LOG_INFO,
"debug": icetray.I3LogLevel.LOG_DEBUG,
"trace": icetray.I3LogLevel.LOG_TRACE
}
if options['LOG_LEVEL'].lower() in log_levels.keys():
icetray.set_log_level(log_levels[options['LOG_LEVEL'].lower()])
else:
logging.warning("log level option %s not recognized.")
logging.warning("Options are ERROR, WARN, INFO, DEBUG, and TRACE.")
logging.warning("Sticking with default of WARN.")
icetray.set_log_level(icetray.I3LogLevel.LOG_WARN)
# make list of input files from GCD and infile
if isinstance(options['infile'], list):
infiles = [options['gcdfile']]
infiles.extend(options['infile'])
else:
infiles = [options['gcdfile'], options['infile']]
# test access to input and output files
for f in infiles:
if not os.access(f, os.R_OK):
raise Exception('Cannot read from %s' % f)
def test_write(f):
if f:
try:
open(f, 'w')
except IOError:
raise Exception('Cannot write to %s' % f)
finally:
os.remove(f)
test_write(options['outfile'])
########################################
tray = I3Tray()
tray.Add(dataio.I3Reader, "reader", filenamelist=infiles)
# run online filters
online_kwargs = {}
if options['photonicsdir']:
online_kwargs.update({
'SplineRecoAmplitudeTable':
os.path.join(options['photonicsdir'], 'splines',
'InfBareMu_mie_abs_z20a10.fits'),
'SplineRecoTimingTable':
os.path.join(options['photonicsdir'], 'splines',
'InfBareMu_mie_prob_z20a10.fits'),
# 'alert_followup_base_GCD_filename': options['gcdfile'],
})
if options['GFU'] is not None:
online_kwargs['gfu_enabled'] = options['GFU']
tray.AddSegment(OnlineFilter,
"OnlineFilter",
decode=False,
simulation=True,
vemcal_enabled=False,
alert_followup=False,
**online_kwargs)
# make random service
seed = os.getpid()
filter_mask_randoms = phys_services.I3GSLRandomService(seed)
# override MinBias Prescale
filterconfigs = filter_globals.filter_pairs + filter_globals.sdst_pairs
if options['MinBiasPrescale']:
for i, filtertuple in enumerate(filterconfigs):
if filtertuple[0] == filter_globals.FilterMinBias:
del filterconfigs[i]
filterconfigs.append(
(filtertuple[0], options['MinBiasPrescale']))
break
icetray.logging.log_info("%s" % str(filterconfigs))
# Generate filter Masks for all P frames
tray.AddModule(filter_tools.FilterMaskMaker,
"MakeFilterMasks",
OutputMaskName=filter_globals.filter_mask,
FilterConfigs=filterconfigs,
RandomService=filter_mask_randoms)
# Merge the FilterMasks
tray.AddModule("OrPframeFilterMasks",
"make_q_filtermask",
InputName=filter_globals.filter_mask,
OutputName=filter_globals.qfilter_mask)
#Q+P frame specific keep module needs to go first, as KeepFromSubstram
#will rename things, let's rename post keep.
def is_Q(frame):
return frame.Stop == frame.DAQ
simulation_keeps = [
'BackgroundI3MCTree',
'BackgroundI3MCTreePEcounts',
'BackgroundI3MCPESeriesMap',
'BackgroundI3MCTree_preMuonProp',
'BackgroundMMCTrackList',
'BeaconLaunches',
'CorsikaInteractionHeight',
'CorsikaWeightMap',
'EventProperties',
'GenerationSpec',
'I3LinearizedMCTree',
'I3MCTree',
'I3MCTreePEcounts',
'I3MCTree_preMuonProp',
'I3MCPESeriesMap',
'I3MCPulseSeriesMap',
'I3MCPulseSeriesMapParticleIDMap',
'I3MCWeightDict',
'LeptonInjectorProperties',
'MCHitSeriesMap',
'MCPrimary',
'MCPrimaryInfo',
'MMCTrackList',
'PolyplopiaInfo',
'PolyplopiaPrimary',
'RNGState',
'SignalI3MCPEs',
'SimTrimmer', # for SimTrimmer flag
'TimeShift', # the time shift amount
'WIMP_params', # Wimp-sim
'noise_weight', # weights for noise-only vuvuzela simulations
'I3GENIEResultDict' # weight informaition for GENIE simulations
]
keep_before_merge = filter_globals.q_frame_keeps + [
'InIceDSTPulses', # keep DST pulse masks
'IceTopDSTPulses',
'CalibratedWaveformRange', # keep calibration info
'UncleanedInIcePulsesTimeRange',
'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange',
'CalibrationErrata',
'SaturationWindows',
'InIceRawData', # keep raw data for now
'IceTopRawData',
] + simulation_keeps
tray.AddModule("Keep",
"keep_before_merge",
keys=keep_before_merge,
If=is_Q)
## second set of prekeeps, conditional on filter content, based on newly created Qfiltermask
#Determine if we should apply harsh keep for events that failed to pass any filter
## Note: excluding the sdst_streams entries
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheckAll",
FilterNameList=filter_globals.filter_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedAnyFilter",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def do_save_just_superdst(frame):
if frame.Has("PassedAnyFilter"):
if not frame["PassedAnyFilter"].value:
return True # <- Event failed to pass any filter.
else:
return False # <- Event passed some filter
else:
icetray.logging.log_error("Failed to find key frame Bool!!")
return False
keep_only_superdsts = filter_globals.keep_nofilterpass + [
'PassedAnyFilter',
'InIceDSTPulses',
'IceTopDSTPulses',
'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange',
'RNGState',
] + simulation_keeps
tray.AddModule("Keep",
"KeepOnlySuperDSTs",
keys=keep_only_superdsts,
If=do_save_just_superdst)
## Now clean up the events that not even the SuperDST filters passed on.
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheckSDST",
FilterNameList=filter_globals.sdst_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedKeepSuperDSTOnly",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def dont_save_superdst(frame):
if frame.Has("PassedKeepSuperDSTOnly") and frame.Has(
"PassedAnyFilter"):
if frame["PassedAnyFilter"].value:
return False # <- these passed a regular filter, keeper
elif not frame["PassedKeepSuperDSTOnly"].value:
return True # <- Event failed to pass SDST filter.
else:
return False # <- Event passed some SDST filter
else:
icetray.logging.log_error("Failed to find key frame Bool!!")
return False
tray.AddModule("Keep",
"KeepOnlyDSTs",
keys=filter_globals.keep_dst_only + [
"PassedAnyFilter", "PassedKeepSuperDSTOnly",
filter_globals.eventheader
],
If=dont_save_superdst)
## Frames should now contain only what is needed. now flatten, write/send to server
## Squish P frames back to single Q frame, one for each split:
tray.AddModule(
"KeepFromSubstream",
"null_stream",
StreamName=filter_globals.NullSplitter,
KeepKeys=filter_globals.null_split_keeps,
)
in_ice_keeps = filter_globals.inice_split_keeps + filter_globals.onlinel2filter_keeps
in_ice_keeps = in_ice_keeps + [
'I3EventHeader', 'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange', 'TriggerSplitterLaunchWindow',
'I3TriggerHierarchy', 'GCFilter_GCFilterMJD'
]
tray.AddModule(
"Keep",
"inice_keeps",
keys=in_ice_keeps,
If=which_split(split_name=filter_globals.InIceSplitter),
)
tray.AddModule(
"KeepFromSubstream",
"icetop_split_stream",
StreamName=filter_globals.IceTopSplitter,
KeepKeys=filter_globals.icetop_split_keeps,
)
# Apply small keep list (SuperDST/SmallTrig/DST/FilterMask for non-filter passers
# Remove I3DAQData object for events not passing one of the 'filters_keeping_allraw'
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheck",
FilterNameList=filter_globals.filters_keeping_allraw,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedConventional",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
## Clean out the Raw Data when not passing conventional filter
def I3RawDataCleaner(frame):
if not (('PassedConventional' in frame
and frame['PassedConventional'].value == True) or
('SimTrimmer' in frame and frame['SimTrimmer'].value == True)):
frame.Delete('InIceRawData')
frame.Delete('IceTopRawData')
tray.AddModule(I3RawDataCleaner,
"CleanErrataForConventional",
Streams=[icetray.I3Frame.DAQ])
###################################################################
########### WRITE STUFF ##########################
###################################################################
# clean up special case for bz2 files to get around empty file bug
bzip2_files = []
if options['outfile'] and options['outfile'].endswith('.bz2'):
options['outfile'] = options['outfile'][:-4]
bzip2_files.append(options['outfile'])
# Write the physics and DAQ frames
tray.AddModule("I3Writer",
"EventWriter",
filename=options['outfile'],
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation
])
# make it go
if options['num'] >= 0:
tray.Execute(options['num'])
else:
tray.Execute()
# print more CPU usage info. than speicifed by default
tray.PrintUsage(fraction=1.0)
for entry in tray.Usage():
stats[entry.key()] = entry.data().usertime
if bzip2_files:
# now do bzip2
if os.path.exists('/usr/bin/bzip2'):
subprocess.check_call(['/usr/bin/bzip2', '-f'] + bzip2_files)
elif os.path.exists('/bin/bzip2'):
subprocess.check_call(['/bin/bzip2', '-f'] + bzip2_files)
else:
raise Exception('Cannot find bzip2')
# clean up forcefully in case we're running this in a loop
del tray
tray.AddModule(
'I3SeededRTCleaning_RecoPulseMask_Module',
'North_seededrt',
InputHitSeriesMapName='SplitInIcePulses',
OutputHitSeriesMapName='SRTInIcePulses',
STConfigService=seededRTConfig,
SeedProcedure='HLCCoreHits',
NHitsThreshold=2,
MaxNIterations=3,
Streams=[icetray.I3Frame.Physics],
# If = which_split(split_name='InIceSplit') & (lambda f: (
# deepcore_wg(f) or wimp_wg(f) or
# muon_wg(f) or cascade_wg(f) or
# ehe_wg(f) or fss_wg(f) ))
If=which_split(split_name='InIceSplit'))
pulses = 'SplitInIcePulses'
tray.AddModule('HomogenizedQTot',
'qtot_total',
Pulses=pulses,
Output='HomogenizedQTot')
# EHE Calibration
tray.AddSegment(
EHECalibration,
'ehecalib',
inPulses='CleanInIceRawData',
outATWD='EHECalibratedATWD_Wave',
outFADC='EHECalibratedFADC_Wave',
# If=lambda f: ehe_wg_Qstream(f)
def Complete(tray,
name,
suffix='',
SplitName='',
SplitPulses='',
FitName='LineFit'):
"""
A segment doing everything that CoincSuite possibly provide.
Run after your splitter, which has written the SplitFrames and SplitCount
:param suffix: right now an unused option FIXME appendable suffix
:param SplitName: name of the subevent-stream, identical to SplitterModule
:param SplitPulses: name of the split PulseSeries in the subevent-stream
:param FitName: Name of that will be given to the LineFit
"""
icetray.logging.log_info("Using CoincSuite Recombinations")
if (SplitName == ''):
icetray.logging.log_fatal(
"Configure CSComplete Traysegment Parameter 'Splitname'")
if (SplitPulses == ''):
icetray.logging.log_fatal(
"Configure CSComplete Traysegment Parameter 'Pulses'")
tray.AddModule(
lambda f: f.Put(SplitName + "ReducedCount", icetray.I3Int(0)),
"ReducedCountMaker",
Streams=[icetray.I3Frame.DAQ])
tray.AddModule("AfterpulseDiscard",
"AfterpulseDiscard",
SplitName=SplitName,
RecoMapName=SplitPulses,
QTotFraction=.1,
TimeOffset=3.E3 * I3Units.ns,
OverlapFraction=0.75,
Discard=True)
tray.AddModule("HypoFrameCreator",
"HypoFrameCreator",
SplitName=SplitName,
HypoName="hypoframe",
RecoMapName=SplitPulses,
MaxTimeSeparation=3000. * I3Units.ns)
from icecube import linefit
tray.AddSegment(linefit.simple,
'LineFit',
inputResponse=SplitPulses,
fitName=FitName,
If=which_split(SplitName) | which_split('hypoframe'))
tray.AddModule(
"TrackSystemTester",
"TestHypoTrackSystem",
SplitName=SplitName,
HypoName="hypoframe",
RecoMapName=SplitPulses,
HypoFitName=FitName,
CriticalRatio=0.7, #0.8
CylinderRadius=150 * I3Units.meter,
ResTimeWindow=dataclasses.make_pair(-200, 200),
ParticleSpeed=dataclasses.I3Constants.c)
tray.AddModule(
"TrackSystemTester",
"TestMutualTrackSystem",
SplitName=SplitName,
HypoName="hypoframe",
RecoMapName=SplitPulses,
RecoFitName=FitName,
CriticalRatio=0.7, #0.8
CylinderRadius=150 * I3Units.meter,
ResTimeWindow=dataclasses.make_pair(-200 * I3Units.ns,
200 * I3Units.ns),
ParticleSpeed=dataclasses.I3Constants.c,
MutualCompare=True)
tray.AddModule("AlignmentTester",
"TestHypoAlignment",
SplitName=SplitName,
HypoName="hypoframe",
HypoFitName=FitName,
RecoFitName=FitName,
CriticalAngle=25 * I3Units.degree,
CriticalDistance=20 * I3Units.meter)
tray.AddModule("AlignmentTester",
"TestMutualAlignment",
SplitName=SplitName,
HypoName="hypoframe",
HypoFitName=FitName,
RecoFitName=FitName,
CriticalAngle=25 * I3Units.degree,
CriticalDistance=20 * I3Units.meter,
MutualCompare=True)
tray.AddModule(
"SpeedTester",
"TestSpeed",
SplitName=SplitName,
HypoName="hypoframe",
HypoFitName=FitName,
SpeedUpperCut=0.35 * I3Units.m / I3Units.ns, #1.17*c
SpeedLowerCut=0.15 * I3Units.m / I3Units.ns) #0.5*c
tray.AddModule("cogCausalConnectTester",
"TestCogCausalConnect",
SplitName=SplitName,
HypoName="hypoframe",
RecoMapName=SplitPulses,
HypoFitName=FitName,
TravelTimeResidual=dataclasses.make_pair(
-1000. * I3Units.ns, 1000. * I3Units.ns),
WallTime=3000. * I3Units.ns,
MaxVerticalDist=700. * I3Units.m,
MaxHorizontalDist=700. * I3Units.m,
MaxTrackDist=200. * I3Units.m,
MaxFurthestDist=600. * I3Units.m)
from icecube import lilliput
import icecube.lilliput.segments
mininame = lilliput.segments.add_minuit_simplex_minimizer_service(tray)
paraname = lilliput.segments.add_simple_track_parametrization_service(tray)
llhname = lilliput.segments.add_pandel_likelihood_service(
tray, SplitPulses, 'SPE1st', 10. * I3Units.hertz)
tray.AddModule("ReducingLikelihoodTester",
"TestHypoReducingLikelihood",
SplitName=SplitName,
HypoName="hypoframe",
HypoFitName=FitName,
RecoFitName=FitName,
LlhName=llhname,
MiniName=mininame,
ParaName=paraname,
ReductionFactor=0.8,
Refit=False)
tray.AddModule("ReducingLikelihoodTester",
"TestMutualReducingLikelihood",
SplitName=SplitName,
HypoName="hypoframe",
HypoFitName=FitName,
RecoFitName=FitName,
LlhName=llhname,
MiniName=mininame,
ParaName=paraname,
ReductionFactor=0.8,
Refit=False,
MutualCompare=True)
tray.AddModule("DecisionMaker",
"FinalDecision",
SplitName=SplitName,
HypoName="hypoframe",
RecoMapName=SplitPulses,
LikeNameList=[
"TestHypoTrackSystem", "TestMutualTrackSystem",
"TestHypoAlignment", "TestMutualAlignment",
"TestHypoReducingLikelihood",
"TestMutualReducingLikelihood"
],
VetoNameList=["TestCogCausalConnect", "TestSpeed"],
TrueNameList=[],
Discard=True)
tray.AddModule(
lambda f: f["I3EventHeader"].sub_event_stream != "hypoframe",
"KillHypoFrame")
tray.AddSegment(linefit.simple,
'LineFit_repeat',
inputResponse=SplitPulses,
fitName=FitName,
If=which_split(SplitName)
& pypick(lambda f: not f.Has(FitName)))
tray.AddModule("Delete", "Delete",
Keys=[]) # NOTE MAYBE remove unused keys here
