from optparse import OptionParser
from os.path import expandvars
import time
import pickle
from I3Tray import *
from icecube import icetray, dataclasses, dataio,neutrinoflux
from icecube import tableio,hdfwriter,DeepCore_Filter,DomTools
from icecube.DeepCore_Filter import DOMS
from icecube.level3_filter_lowen import retro_level3_segments
import L4Segments
icetray.set_log_level(icetray.I3LogLevel.LOG_WARN)
#----------------------------------------------------------------------
# Allows the parser to take lists as inputs. Cleans out any white
# spaces that may be included in the names of the Official or
# Personal scripts to be run. Also removes the .py which is required
# to load a python module.
def list_callback(option, opt, value, parser):
noWhiteSpace = value.replace(' ', '')
noPyAppend = noWhiteSpace.replace('.py', '')
cleanList = noPyAppend.split(',')
setattr(parser.values, option.dest, cleanList)
def string_callback(option, opt, value, parser):
lower_case = value.lower()
import icecube
from I3Tray import *
from icecube import icetray, dataclasses, dataio
from icecube.gulliver import I3LogLikelihoodFitParams
from icecube.weighting.fluxes import GaisserH4a
from icecube.weighting.weighting import from_simprod
from icecube.icetray import I3Units
icetray.set_log_level(icetray.I3LogLevel.LOG_FATAL)
#vars
import numpy as np
import glob
import sys
import argparse
import re
parser = argparse.ArgumentParser()
parser.add_argument("-o",
"--outfile",
dest="outfile",
type=str,
default="Out",
help="base name for outfile")
args = parser.parse_args()
outfile = args.outfile
#dsets = {12040:23346,12161:98026,12268:99985,12332:98370,12379:34516,
# 9036:99977,9255:99973,9622:99978,
# 20787:98557,20789:80272,20848:98933,20852:95332,20849:9948}
parser.add_argument("-o", "--outfile", help="path for outfile", type=str)
parser.add_argument("-c", "--configfile", help="path for outfile", type=str)
args = parser.parse_args()
print "check access"
os.path.exists(os.path.expandvars(args.gcdfile))
os.path.exists(os.path.expandvars(args.infile))
os.path.exists(os.path.expandvars(args.configfile))
os.access(args.outfile, os.W_OK)
from icecube import IceHiveZ
#now lets see if we can load the saved connectorblock
print "load"
cb = IceHiveZ.ConnectorBlock.from_file(args.configfile)
icetray.set_log_level(icetray.I3LogLevel.LOG_INFO)
#icetray.set_log_level_for_unit("I3HiveSplitter", icetray.I3LogLevel.LOG_DEBUG)
#icetray.set_log_level_for_unit("HiveSplitter", icetray.I3LogLevel.LOG_DEBUG)
#icetray.set_log_level_for_unit("I3HiveCleaning", icetray.I3LogLevel.LOG_DEBUG)
#icetray.set_log_level_for_unit("HiveCleaning", icetray.I3LogLevel.LOG_DEBUG)
tray = I3Tray()
#read some file
tray.AddModule("I3Reader", "Reader", FilenameList=[args.gcdfile, args.infile])
tray.AddModule("I3HiveClustering",
"HiveClustering",
Stream=icetray.I3Frame.Physics,
InputName="SRTTWOfflinePulses",
OutputName="HiveSRTTWOfflinePulses",
def PerformPropagationSimulation():
tray = I3Tray()
tray.AddModule("I3Reader", FilenameList=input_files)
tray.AddService("I3SPRNGRandomServiceFactory",
Seed=options.seed,
NStreams=2,
StreamNum=1)
randomService = phys_services.I3SPRNGRandomService(seed=options.seed,
nstreams=10000,
streamnum=1)
common_clsim_parameters = dict(
PhotonSeriesName="PropagatedPhotons",
RandomService=randomService,
UseGPUs=options.use_gpus,
UseCPUs=not options.use_gpus,
ParallelEvents=options.number_of_parallel_runs,
IceModelLocation=options.ice_model_file,
UnWeightedPhotons=True,
#UnWeightedPhotonsScalingFactor=0.01,
DOMOversizeFactor=1.0,
UnshadowedFraction=1.0,
FlasherInfoVectName=("I3FlasherInfo"
if options.use_flasher_info_vect else ""),
FlasherPulseSeriesName=("PhotonFlasherPulseSeries"
if not options.use_flasher_info_vect else ""),
UseHoleIceParameterization=options.
use_hole_ice_approximation, # Angular Acceptance Modification.
)
# In production, we do not need to save the photons. Thus, we skip this for
# performance and disk space reasons. But one can activate
# `--save-photon-paths` for visualization or debug reasons.
#
if options.save_photon_paths:
tray.AddSegment(
clsim.I3CLSimMakePhotons,
StopDetectedPhotons=False,
PhotonHistoryEntries=10000,
ExtraArgumentsToI3CLSimModule=dict(
SaveAllPhotons=True,
SaveAllPhotonsPrescale=1.0,
MaxNumOutputPhotonsCorrectionFactor=1.0,
SimulateHoleIce=options.use_hole_ice_simulation,
HoleIceScatteringLengthFactor=options.scattering_factor,
HoleIceAbsorptionLengthFactor=options.absorption_factor),
**common_clsim_parameters)
icetray.set_log_level(icetray.I3LogLevel.LOG_DEBUG)
else:
extra_args = dict()
if options.use_hole_ice_simulation:
extra_args = dict(
SimulateHoleIce=options.use_hole_ice_simulation,
HoleIceScatteringLengthFactor=options.scattering_factor,
HoleIceAbsorptionLengthFactor=options.absorption_factor)
tray.AddSegment(clsim.I3CLSimMakeHits,
StopDetectedPhotons=True,
ExtraArgumentsToI3CLSimModule=extra_args,
**common_clsim_parameters)
if not options.save_photon_paths:
# This does only work with the `I3CLSimMakeHits` module, thus does
# not work with `save_photon_paths`.
tray.AddModule(ReadOutAngularAcceptance, Streams=[icetray.I3Frame.DAQ])
tray.AddModule("I3Writer", Filename=options.output_i3_file)
tray.AddModule("TrashCan")
if options.number_of_frames == 0:
tray.Execute()
else:
tray.Execute(options.number_of_frames)
tray.Finish()
#!/usr/bin/env python
import sys
from I3Tray import *
from icecube import icetray, dataclasses, dataio, phys_services, sim_services, hdfwriter, tableio
from icecube import SNPS
tray = I3Tray()
icetray.set_log_level(icetray.I3LogLevel.LOG_INFO)
tray.AddModule("I3Reader",
"reader",
FilenameList=sys.argv[2:])
tray.AddModule("SNOscillations", "Oscillations",
)
tray.AddModule("I3NullSplitter", 'nullsplit')
tray.AddModule("SNPSDataCollector", "I3FileBooking",
InputMCPESeriesName = "MCPESeriesMap",
InputPhotonSeriesName = "PhotonSeriesMap",
InputMCTreeName = "I3MCTree",
InputEventHeaderName = "I3EventHeader",
InputSNWeightName="SNWeightMap",
OutputMCPEFilterName = "MCPE_Filtered",
#!/usr/bin/env python
#
# pybinding load test
#
from I3Tray import *
from icecube import icetray, dataclasses, earthmodel_service
#
# for logging
#
icetray.I3Logger.global_logger = icetray.I3PrintfLogger()
icetray.set_log_level(icetray.I3LogLevel.LOG_WARN)
#icetray.set_log_level_for_unit("EarthModelService",icetray.I3LogLevel.LOG_TRACE)
#
# generate earthmodel service
#
print("*** flat density integration test ***")
earthmodel = ["PREM_mmc", "FLATCORE_5.568"]
earth1 = earthmodel_service.EarthModelService("EarthModelService","",
earthmodel)
r = 3480000;
orig = dataclasses.I3Position(0, 0, 0);
to = dataclasses.I3Position(0, 0, r);
pass1 = earth1.integrate_density_in_cgs(orig, to, earthmodel_service.path) ;
pass2 = earth1.integrate_density_in_cgs(orig, to, earthmodel_service.radius) ;
import sys
from icecube import icetray, dataio
from icecube.icetray import I3Units
from icecube import STTools
from icecube.STTools.seededRT.configuration_services import I3DOMLinkSeededRTConfigurationService
icetray.set_log_level(icetray.I3LogLevel.LOG_TRACE)
icetray.set_log_level_for_unit('STToolsSeededRT', icetray.I3LogLevel.LOG_TRACE)
icetray.set_log_level_for_unit('OMKeyPairMap', icetray.I3LogLevel.LOG_DEBUG)
icetray.set_log_level_for_unit('OMKeyHasher', icetray.I3LogLevel.LOG_DEBUG)
icetray.set_log_level_for_unit('I3SeededRTConfigurationService',
icetray.I3LogLevel.LOG_DEBUG)
try:
gcdfilename = sys.argv[1]
infilename = sys.argv[2]
outfilename = sys.argv[3]
except:
raise RuntimeError("Usage: %s gcdfile infile outfile" % (sys.argv[0]))
# Create a seededRT configuration service, based on DOM link ST settings.
seededRTConfigService = I3DOMLinkSeededRTConfigurationService(
allowSelfCoincidence=False, # Default: False
useDustlayerCorrection=True, # Default: True
treat_string_36_as_deepcore=True, # Default: True
dustlayerUpperZBoundary=0 * I3Units.m,
dustlayerLowerZBoundary=-150 * I3Units.m,
ic_ic_RTTime=1000 * I3Units.ns,
ic_ic_RTRadius=150 * I3Units.m)
UseCPUs = args.cpu
UseGPUs = not args.cpu
DOMOversizeFactor = args.domoversizefactor
UseI3PropagatorService = args.UseI3PropagatorService
log_level = args.log_level
# set icetray logging level
log_levels = {
"error": icetray.I3LogLevel.LOG_ERROR,
"warn": icetray.I3LogLevel.LOG_WARN,
"info": icetray.I3LogLevel.LOG_INFO,
"debug": icetray.I3LogLevel.LOG_DEBUG,
"trace": icetray.I3LogLevel.LOG_TRACE
}
if log_level.lower() in log_levels.keys():
icetray.set_log_level(log_levels[log_level.lower()])
# read either yaml or json config file for Snowstorm
print("Reading Snowstorm config... ", end="")
_, ext = os.path.splitext(args.config_file)
if ext == ".json":
loader = json.load
elif ext in [".yaml", ".yml"]:
loader = yaml.load
else:
raise ValueError(
"Cannot load snowstorm config with extension {}".format(ext))
Snowstorm_config = loader(open(args.config_file))
print("done")
def oscNext_L7(
tray, name,
uncleaned_pulses,
cleaned_pulses,
reco_particle=RETRO_RECO_PARTICLE_KEY,
debug=False,
):
'''
This is the main oscNext L7 traysegment
'''
if debug:
icetray.set_log_level(icetray.I3LogLevel.LOG_DEBUG)
else:
icetray.set_log_level(icetray.I3LogLevel.LOG_ERROR)
#
# Data quality
#
# Check the required pulse series are present
tray.Add(check_object_exists,object_key=uncleaned_pulses)
tray.Add(check_object_exists,object_key=cleaned_pulses)
#
# Apply cuts on input data
#
# Only keep frames passing L6
tray.Add(oscNext_cut, "L6_cut", processing_level=6)
# Only keep frames where retro reco was successful
#TODO This is now part of L6 but some files were run before this
tray.Add(check_retro_reco_success, "check_retro_reco_success")
#
# Compute L7 variables
#
# Get final reco values
tray.Add(
oscNext_L7_compute_final_reconstructed_values,
'oscNext_L7_compute_final_reconstructed_values',
cleaned_pulses=cleaned_pulses,
)
# Find coincident events (unsimulated)
#tray.Add(
# oscNext_L7_CoincidentCut,
# 'oscNext_L7_CoincidentCut',
# uncleaned_pulses=uncleaned_pulses,
# cleaned_pulses=cleaned_pulses,
#)
# Compare to older samples
tray.Add(
oscNext_L7_compute_comparisons_to_older_samples,
'oscNext_L7_compute_comparisons_to_older_samples',
)
# Compare different reconstructions
tray.Add(
oscNext_L7_compute_reconstruction_comparisons,
'oscNext_L7_compute_reconstruction_comparisons',
)
# Calculate STV Variables
#tray.Add(
# oscNext_L7_STV,
# "oscNext_L7_STV_InfTrack_DC",
# uncleaned_pulses=uncleaned_pulses,
# cleaned_pulses=cleaned_pulses,
# reco_particle=reco_particle,
reco_shape=dataclasses.I3Particle.InfiniteTrack, # Primary is the default shape of retro-reco particles
include_DeepCore = True,
def oscNext_L7(
tray,
name,
uncleaned_pulses,
cleaned_pulses,
reco_particle=RETRO_RECO_PARTICLE_KEY,
debug=False,
):
'''
This is the main oscNext L7 traysegment
'''
if debug:
icetray.set_log_level(icetray.I3LogLevel.LOG_DEBUG)
else:
icetray.set_log_level(icetray.I3LogLevel.LOG_ERROR)
#
# Data quality
#
# Check the required pulse series are present
tray.Add(check_object_exists, object_key=uncleaned_pulses)
tray.Add(check_object_exists, object_key=cleaned_pulses)
# Make the L3 data quality cuts
# Update on 19th August 2019: This applies SLOP and flaring DOM cuts which have not been made before
#TODO Can remove this in the future once have re-processed everything (e.g. because it will then have already been run at L3)
from icecube.level3_filter_lowen.LowEnergyL3TraySegment import DataQualityCuts
tray.AddSegment(DataQualityCuts,
"L3_DataQualityCuts",
pulses=cleaned_pulses,
make_cut=True,
If=lambda frame: "Data_quality_bool" not in frame)
#
# Apply cuts on input data
#
# Only keep frames passing L6
tray.Add(oscNext_cut, "L6_cut", processing_level=6)
# Only keep frames where retro reco was successful
#TODO This is now part of L6 but some files were run before this
tray.Add(check_retro_reco_success, "check_retro_reco_success")
#
# Compute L7 variables
#
# Get final reco values
tray.Add(
oscNext_L7_compute_final_reconstructed_values,
'oscNext_L7_compute_final_reconstructed_values',
cleaned_pulses=cleaned_pulses,
)
# Find coincident events (unsimulated)
tray.Add(
oscNext_L7_CoincidentCut,
'oscNext_L7_CoincidentCut',
uncleaned_pulses=uncleaned_pulses,
cleaned_pulses=cleaned_pulses,
)
# Compare to older samples
tray.Add(
oscNext_L7_compute_comparisons_to_older_samples,
'oscNext_L7_compute_comparisons_to_older_samples',
)
# Compare different reconstructions
tray.Add(
oscNext_L7_compute_reconstruction_comparisons,
'oscNext_L7_compute_reconstruction_comparisons',
)
# Calculate STV Variables
tray.Add(
oscNext_L7_STV,
"oscNext_L7_STV_InfTrack_DC",
uncleaned_pulses=uncleaned_pulses,
cleaned_pulses=cleaned_pulses,
reco_particle=reco_particle,
reco_shape=dataclasses.I3Particle.
InfiniteTrack, # Primary is the default shape of retro-reco particles
include_DeepCore=True,
)
# Corridor cut
# Compute a version of the corridor cut with much narroweer corridor definitions compared to L5
# This is more like what was done in DRAGON/GRECO
tray.Add(
oscNext_L7_CorridorCuts,
'oscNext_L7_NarrowCorrCuts',
uncleaned_pulses=uncleaned_pulses,
cleaned_pulses=cleaned_pulses,
reco_particle=reco_particle,
)
# Photon speeds
tray.Add(
oscNext_L7_photon_speed_metrics,
"oscNext_L7_photon_speed_metrics",
cleaned_pulses=cleaned_pulses,
)
# PID
tray.Add(
oscNext_L7_compute_pid,
"oscNext_L7_PID",
)
#
# L7 cut
#
# Compute cut
tray.Add(
oscNext_L7_compute_cut,
"oscNext_L7_cut",
)
#
# Done
#
# Can dump frame for debugging
if False:
tray.Add("Dump", "Dump")
# Easier reading
if debug:
def add_line_break(frame):
print "\n*********************\n"
tray.Add(add_line_break)
return
# ###############################
import sys
from I3Tray import *
from icecube import icetray, dataclasses, dataio, phys_services, sim_services
from icecube import clsim
from icecube import SNPS
import math, numpy
tray = I3Tray()
icetray.set_log_level(icetray.I3LogLevel.LOG_DEBUG)
tray.AddModule("I3Reader",
"reader",
FilenameList=sys.argv[1])
tray.AddModule("SNFilePruner", "Pruner",
NumPrimariesPerFrame=1001)
# write the results to disk
tray.AddModule("I3Writer", "writer",
filename=sys.argv[1][:-7] + "_pruned.i3.bz2",
Streams=[icetray.I3Frame.DAQ, icetray.I3Frame.TrayInfo])
import sys, math, os
from I3Tray import *
from icecube import icetray, dataclasses, dataio, phys_services
from icecube import simclasses
from icecube import DOMLauncher
tray = I3Tray()
i3_testdata = os.path.expandvars("$I3_TESTDATA")
tray.AddModule("I3InfiniteSource",
"FrameMaker",
Prefix=i3_testdata +
'/sim/GeoCalibDetectorStatus_IC86.55697_corrected_V2.i3.gz')
icetray.set_log_level(icetray.I3LogLevel.LOG_DEBUG)
import unittest
#This test module is testing the basics of the LC logic. Testing the LC time window for instance.
class SanityCheck(unittest.TestCase):
def __init__(self, foobar):
super(SanityCheck, self).__init__(foobar)
#Setting up the globals for the DOMObject.
self.random_service = phys_services.I3GSLRandomService(2)
def testSLC(self):
geo = self.frame["I3Geometry"].omgeo
cal = self.frame["I3Calibration"].dom_cal
stat = self.frame["I3DetectorStatus"].dom_status
def main(options, stats={}):
"""The main L1 script"""
tray = I3Tray()
if not options['infile']:
raise Exception(
'You need to specify an input file with the -i or --input parameter.'
)
if not options['gcdfile']:
raise Exception(
'You need to specify a GCD file with the -g or --gcd parameter.')
if not options['outfile']:
raise Exception(
'You need to specify an output file with the -o or --output parameter.'
)
log_levels = {
"error": icetray.I3LogLevel.LOG_ERROR,
"warn": icetray.I3LogLevel.LOG_WARN,
"info": icetray.I3LogLevel.LOG_INFO,
"debug": icetray.I3LogLevel.LOG_DEBUG,
"trace": icetray.I3LogLevel.LOG_TRACE
}
if options['LOG_LEVEL'].lower() in log_levels.keys():
icetray.set_log_level(log_levels[options['LOG_LEVEL'].lower()])
else:
logging.warning("log level option %s not recognized.")
logging.warning("Options are ERROR, WARN, INFO, DEBUG, and TRACE.")
logging.warning("Sticking with default of WARN.")
icetray.set_log_level(icetray.I3LogLevel.LOG_WARN)
# make list of input files from GCD and infile
if isinstance(options['infile'], list):
infiles = [options['gcdfile']]
infiles.extend(options['infile'])
else:
infiles = [options['gcdfile'], options['infile']]
# test access to input and output files
for f in infiles:
if not os.access(f, os.R_OK):
raise Exception('Cannot read from %s' % f)
def test_write(f):
if f:
try:
open(f, 'w')
except IOError:
raise Exception('Cannot write to %s' % f)
finally:
os.remove(f)
test_write(options['outfile'])
########################################
tray = I3Tray()
tray.Add(dataio.I3Reader, "reader", filenamelist=infiles)
# run online filters
online_kwargs = {}
if options['photonicsdir']:
online_kwargs.update({
'SplineRecoAmplitudeTable':
os.path.join(options['photonicsdir'], 'splines',
'InfBareMu_mie_abs_z20a10.fits'),
'SplineRecoTimingTable':
os.path.join(options['photonicsdir'], 'splines',
'InfBareMu_mie_prob_z20a10.fits'),
# 'alert_followup_base_GCD_filename': options['gcdfile'],
})
if options['GFU'] is not None:
online_kwargs['gfu_enabled'] = options['GFU']
tray.AddSegment(OnlineFilter,
"OnlineFilter",
decode=False,
simulation=True,
vemcal_enabled=False,
alert_followup=False,
**online_kwargs)
# make random service
seed = os.getpid()
filter_mask_randoms = phys_services.I3GSLRandomService(seed)
# override MinBias Prescale
filterconfigs = filter_globals.filter_pairs + filter_globals.sdst_pairs
if options['MinBiasPrescale']:
for i, filtertuple in enumerate(filterconfigs):
if filtertuple[0] == filter_globals.FilterMinBias:
del filterconfigs[i]
filterconfigs.append(
(filtertuple[0], options['MinBiasPrescale']))
break
icetray.logging.log_info("%s" % str(filterconfigs))
# Generate filter Masks for all P frames
tray.AddModule(filter_tools.FilterMaskMaker,
"MakeFilterMasks",
OutputMaskName=filter_globals.filter_mask,
FilterConfigs=filterconfigs,
RandomService=filter_mask_randoms)
# Merge the FilterMasks
tray.AddModule("OrPframeFilterMasks",
"make_q_filtermask",
InputName=filter_globals.filter_mask,
OutputName=filter_globals.qfilter_mask)
#Q+P frame specific keep module needs to go first, as KeepFromSubstram
#will rename things, let's rename post keep.
def is_Q(frame):
return frame.Stop == frame.DAQ
simulation_keeps = [
'BackgroundI3MCTree',
'BackgroundI3MCTreePEcounts',
'BackgroundI3MCPESeriesMap',
'BackgroundI3MCTree_preMuonProp',
'BackgroundMMCTrackList',
'BeaconLaunches',
'CorsikaInteractionHeight',
'CorsikaWeightMap',
'EventProperties',
'GenerationSpec',
'I3LinearizedMCTree',
'I3MCTree',
'I3MCTreePEcounts',
'I3MCTree_preMuonProp',
'I3MCPESeriesMap',
'I3MCPulseSeriesMap',
'I3MCPulseSeriesMapParticleIDMap',
'I3MCWeightDict',
'LeptonInjectorProperties',
'MCHitSeriesMap',
'MCPrimary',
'MCPrimaryInfo',
'MMCTrackList',
'PolyplopiaInfo',
'PolyplopiaPrimary',
'RNGState',
'SignalI3MCPEs',
'SimTrimmer', # for SimTrimmer flag
'TimeShift', # the time shift amount
'WIMP_params', # Wimp-sim
'noise_weight', # weights for noise-only vuvuzela simulations
'I3GENIEResultDict' # weight informaition for GENIE simulations
]
keep_before_merge = filter_globals.q_frame_keeps + [
'InIceDSTPulses', # keep DST pulse masks
'IceTopDSTPulses',
'CalibratedWaveformRange', # keep calibration info
'UncleanedInIcePulsesTimeRange',
'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange',
'CalibrationErrata',
'SaturationWindows',
'InIceRawData', # keep raw data for now
'IceTopRawData',
] + simulation_keeps
tray.AddModule("Keep",
"keep_before_merge",
keys=keep_before_merge,
If=is_Q)
## second set of prekeeps, conditional on filter content, based on newly created Qfiltermask
#Determine if we should apply harsh keep for events that failed to pass any filter
## Note: excluding the sdst_streams entries
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheckAll",
FilterNameList=filter_globals.filter_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedAnyFilter",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def do_save_just_superdst(frame):
if frame.Has("PassedAnyFilter"):
if not frame["PassedAnyFilter"].value:
return True # <- Event failed to pass any filter.
else:
return False # <- Event passed some filter
else:
icetray.logging.log_error("Failed to find key frame Bool!!")
return False
keep_only_superdsts = filter_globals.keep_nofilterpass + [
'PassedAnyFilter',
'InIceDSTPulses',
'IceTopDSTPulses',
'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange',
'RNGState',
] + simulation_keeps
tray.AddModule("Keep",
"KeepOnlySuperDSTs",
keys=keep_only_superdsts,
If=do_save_just_superdst)
## Now clean up the events that not even the SuperDST filters passed on.
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheckSDST",
FilterNameList=filter_globals.sdst_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedKeepSuperDSTOnly",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def dont_save_superdst(frame):
if frame.Has("PassedKeepSuperDSTOnly") and frame.Has(
"PassedAnyFilter"):
if frame["PassedAnyFilter"].value:
return False # <- these passed a regular filter, keeper
elif not frame["PassedKeepSuperDSTOnly"].value:
return True # <- Event failed to pass SDST filter.
else:
return False # <- Event passed some SDST filter
else:
icetray.logging.log_error("Failed to find key frame Bool!!")
return False
tray.AddModule("Keep",
"KeepOnlyDSTs",
keys=filter_globals.keep_dst_only + [
"PassedAnyFilter", "PassedKeepSuperDSTOnly",
filter_globals.eventheader
],
If=dont_save_superdst)
## Frames should now contain only what is needed. now flatten, write/send to server
## Squish P frames back to single Q frame, one for each split:
tray.AddModule(
"KeepFromSubstream",
"null_stream",
StreamName=filter_globals.NullSplitter,
KeepKeys=filter_globals.null_split_keeps,
)
in_ice_keeps = filter_globals.inice_split_keeps + filter_globals.onlinel2filter_keeps
in_ice_keeps = in_ice_keeps + [
'I3EventHeader', 'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange', 'TriggerSplitterLaunchWindow',
'I3TriggerHierarchy', 'GCFilter_GCFilterMJD'
]
tray.AddModule(
"Keep",
"inice_keeps",
keys=in_ice_keeps,
If=which_split(split_name=filter_globals.InIceSplitter),
)
tray.AddModule(
"KeepFromSubstream",
"icetop_split_stream",
StreamName=filter_globals.IceTopSplitter,
KeepKeys=filter_globals.icetop_split_keeps,
)
# Apply small keep list (SuperDST/SmallTrig/DST/FilterMask for non-filter passers
# Remove I3DAQData object for events not passing one of the 'filters_keeping_allraw'
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheck",
FilterNameList=filter_globals.filters_keeping_allraw,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedConventional",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
## Clean out the Raw Data when not passing conventional filter
def I3RawDataCleaner(frame):
if not (('PassedConventional' in frame
and frame['PassedConventional'].value == True) or
('SimTrimmer' in frame and frame['SimTrimmer'].value == True)):
frame.Delete('InIceRawData')
frame.Delete('IceTopRawData')
tray.AddModule(I3RawDataCleaner,
"CleanErrataForConventional",
Streams=[icetray.I3Frame.DAQ])
###################################################################
########### WRITE STUFF ##########################
###################################################################
# clean up special case for bz2 files to get around empty file bug
bzip2_files = []
if options['outfile'] and options['outfile'].endswith('.bz2'):
options['outfile'] = options['outfile'][:-4]
bzip2_files.append(options['outfile'])
# Write the physics and DAQ frames
tray.AddModule("I3Writer",
"EventWriter",
filename=options['outfile'],
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation
])
# make it go
if options['num'] >= 0:
tray.Execute(options['num'])
else:
tray.Execute()
# print more CPU usage info. than speicifed by default
tray.PrintUsage(fraction=1.0)
for entry in tray.Usage():
stats[entry.key()] = entry.data().usertime
if bzip2_files:
# now do bzip2
if os.path.exists('/usr/bin/bzip2'):
subprocess.check_call(['/usr/bin/bzip2', '-f'] + bzip2_files)
elif os.path.exists('/bin/bzip2'):
subprocess.check_call(['/bin/bzip2', '-f'] + bzip2_files)
else:
raise Exception('Cannot find bzip2')
# clean up forcefully in case we're running this in a loop
del tray
copy_to_OSG(gcdfile_NPX)
copy_to_OSG(infile_NPX)
#copy_to_OSG('gsiftp://gridftp.icecube.wisc.edu/data/user/twatson/Ice/build/CLSimModels.i3')
#os.rename(str(os.getcwd()) + "/CLSimModels.i3", str(os.getcwd()) + "/MultiSimBuild.RHEL_6_x86_64/CLSimModels.i3")
print gcdfile
infiles = [gcdfile,infile]
outfile = str('gsiftp://gridftp.icecube.wisc.edu' + outfile)
outfile_temp = str(os.getcwd()+'/'+outfile.split('/')[-1])
else:
infiles = [gcdfile, infile]
outfile_temp = '/data/ana/SterileNeutrino/IC86/HighEnergy/MC/scripts/temp/'+outfile.split('/')[-1]
tray = I3Tray()
icetray.set_log_level(icetray.I3LogLevel.LOG_ERROR)
print('Making RNGs.')
randomServicePhotons = phys_services.I3GSLRandomService(seed = int(seed))#SPENCER -- I set the seed back to what it was to keep it consistent with old MC.
randomServicePropagators = phys_services.I3GSLRandomService(seed = int(seed))
tray.AddModule("I3Reader", "reader",filenamelist=infiles)
tray.AddModule("Rename",keys=["I3MCTree","I3MCTree_preMuonProp"])
tray.AddSegment(segments.PropagateMuons, "PropagateMuons",
RandomService = randomServicePropagators)
tray.context['I3RandomService'] = randomServicePhotons
print('Making Photons.')
tray.AddSegment(clsim.I3CLSimMakePhotons, "MakingPhotons",
UseCPUs = usecpus,
UseGPUs = usegpus,
UseOnlyDeviceNumber = None,
def main(options, stats={}):
"""The main L2 processing script"""
tray = I3Tray()
log_levels = {"error" : icetray.I3LogLevel.LOG_ERROR,
"warn" : icetray.I3LogLevel.LOG_WARN,
"info" : icetray.I3LogLevel.LOG_INFO,
"debug" : icetray.I3LogLevel.LOG_DEBUG,
"trace" : icetray.I3LogLevel.LOG_TRACE}
if options['LOG_LEVEL'].lower() in log_levels.keys():
icetray.set_log_level(log_levels[options['LOG_LEVEL'].lower()])
else:
print("log level option %s not recognized.")
print("Options are ERROR, WARN, INFO, DEBUG, and TRACE.")
print("Sticking with default of WARN.")
icetray.set_log_level(icetray.I3LogLevel.LOG_WARN)
# make list of input files from GCD and infile
if isinstance(options['infile'],list):
infiles = [options['gcdfile']]
infiles.extend(options['infile'])
else:
infiles = [options['gcdfile'], options['infile']]
print('infiles: ',infiles)
# test access to input and output files
for f in infiles:
if not os.access(f,os.R_OK):
raise Exception('Cannot read from %s'%f)
def test_write(f):
if f:
try:
open(f,'w')
except IOError:
raise Exception('Cannot write to %s'%f)
finally:
os.remove(f)
test_write(options['outfile'])
test_write(options['dstfile'])
test_write(options['gapsfile'])
test_write(options['icetopoutput'])
test_write(options['eheoutput'])
test_write(options['slopoutput'])
test_write(options['rootoutput'])
# read input files
tray.Add( dataio.I3Reader, "Reader",
Filenamelist = infiles
)
tray.AddSegment(OfflineFilter, "OfflineFilter",
dstfile=options['dstfile'],
mc=options['mc'],
doNotQify=options['mc'],
photonicsdir=options['photonicsdir']
)
###################################################################
########### WRITE STUFF ##########################
###################################################################
# clean up special case for bz2 files to get around empty file bug
bzip2_files = []
if options['outfile'] and options['outfile'].endswith('.bz2'):
options['outfile'] = options['outfile'][:-4]
bzip2_files.append(options['outfile'])
if options['dstfile'] and options['dstfile'].endswith('.bz2'):
options['dstfile'] = options['dstfile'][:-4]
bzip2_files.append(options['dstfile'])
if options['icetopoutput'] and options['icetopoutput'].endswith('.bz2'):
options['icetopoutput'] = options['icetopoutput'][:-4]
bzip2_files.append(options['icetopoutput'])
if options['eheoutput'] and options['eheoutput'].endswith('.bz2'):
options['eheoutput'] = options['eheoutput'][:-4]
bzip2_files.append(options['eheoutput'])
if options['slopoutput'] and options['slopoutput'].endswith('.bz2'):
options['slopoutput'] = options['slopoutput'][:-4]
bzip2_files.append(options['slopoutput'])
tray.AddModule( "I3Writer", "EventWriter" ,
Filename = options['outfile'],
Streams = [icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation],
DropOrphanStreams = [icetray.I3Frame.DAQ],
)
# special outputs
if options['eheoutput']:
tray.AddSegment(SpecialWriter.EHEWriter, "write_ehe",
Filename=options['eheoutput']
)
if options['slopoutput']:
tray.AddSegment(SpecialWriter.SLOPWriter, "write_slop",
Filename=options['slopoutput']
)
if options['rootoutput']:
tray.AddSegment(SpecialWriter.RootWriter, "write_root",
Filename = options['rootoutput']
)
if options['gapsfile']:
tray.AddSegment(SpecialWriter.GapsWriter, "write_gaps",
Filename = options['gapsfile'],
MinGapTime = 1
)
if options['icetopoutput']:
# this needs to be the last output
tray.AddModule('Delete','deleteicetopextrakeys',
keys=['InIceRawData','CleanInIceRawData']
)
tray.AddSegment(SpecialWriter.IceTopWriter, "write_icetop",
Filename=options['icetopoutput']
)
# make it go
if options['num'] >= 0:
tray.Execute(options['num'])
else:
tray.Execute()
# print more CPU usage info. than speicifed by default
tray.PrintUsage(fraction=1.0)
for entry in tray.Usage():
stats[entry.key()] = entry.data().usertime
if bzip2_files:
# now do bzip2
if os.path.exists('/usr/bin/bzip2'):
subprocess.check_call(['/usr/bin/bzip2', '-f']+bzip2_files)
elif os.path.exists('/bin/bzip2'):
subprocess.check_call(['/bin/bzip2', '-f']+bzip2_files)
else:
raise Exception('Cannot find bzip2')
# clean up forcefully in case we're running this in a loop
del tray
