def ExtractDST13(tray, name,
dst_output_filename = "I3DST13.hdf5",
dstname = "I3DST13",
extract_to_frame = True,
If = lambda f: True):
"""
Record in compact form limited information from reconstructions, triggers and cut
parameters for every triggered event.
"""
from icecube import dst
from icecube import phys_services
from icecube.tableio import I3TableWriter
from . import TDSTConverter
# Open output file
if dst_output_filename.endswith('.root'):
# Open output file
from icecube.rootwriter import I3ROOTTableService
table_service = I3ROOTTableService(filename= dst_output_filename,
master= "dst", #Default name: "MasterTree".
#mode=RECREATE,
)
elif dst_output_filename.endswith('.hdf5') or dst_output_filename.endswith('.hd5'):
from icecube.hdfwriter import I3HDFTableService
table_service = I3HDFTableService(dst_output_filename, 6)
if "I3RandomService" not in tray.context:
dstRng = phys_services.I3GSLRandomService(int(time.time()))
tray.context["I3RandomService"]=dstRng
tray.AddModule('I3DSTExtractor13', 'UnpackDST',
SubEventStreamName = 'TDST13',
FileName = dst_output_filename,
DSTName = dstname,
DSTHeaderName = dstname+"Header",
EventHeaderName = 'I3EventHeader',
ExtractToFrame = extract_to_frame,
TriggerName = 'I3TriggerHierarchy',
)
tray.AddModule(I3TableWriter, "writer",
TableService = table_service,
SubEventStreams= ['TDST13'],
Keys = [ "CutDST", "TDSTTriggers"],
)
def convert(inputpath,
outputfile,
file_type='hdf5',
sub_event_stream='InIceSplit',
verbose=False,
generateID=False):
'''
Convert files from i3 to hdf5 or root
Args:
inputpath: Inputpath to be scanned for i3 files, input file works, too
outputfile: Outputpath for the converted file
file_type: Choose the output format between root and hdf5
sub_event_stream: Provide the i3 subeventstream to use
verbose: Provide verbose output
generateID: Generate a new unique event id
Returns:
Nothing
'''
# give output if requested
if verbose:
print 'Input folder is "', inputpath
print 'Output file is "', outputfile
print 'Outputformat is "', file_type
if generateID:
print 'P-frame-based ID will be added to "I3EventHeader".'
# list of possible i3 endings
i3_endings = ['i3', 'i3.gz', 'i3.bz2']
tray = I3Tray()
# create list of all files of input path
i3_files = []
# differentiate between a given path or filename
if os.path.isdir(inputpath):
root, subdirs, _ = os.walk(inputpath).next()
if verbose:
print("Subdirs:")
print(subdirs)
print("Filenames:")
for subdir in subdirs:
for filename in os.listdir(os.path.join(root, subdir)):
if verbose:
print(filename)
if any([filename.endswith(ending) for ending in i3_endings]):
i3_files.append(os.path.join(*[root, subdir, filename]))
tray.AddModule('I3Reader', 'reader', FilenameList=i3_files)
else:
# seems to be a single file
outputfile = os.path.join(
os.path.dirname(outputfile),
os.path.basename(inputpath[:inputpath.find('.i3')]))
print(outputfile)
if any([inputpath.endswith(ending) for ending in i3_endings]):
tray.AddModule('I3Reader', 'reader', Filename=inputpath)
else:
print('File format not supported')
# create output path if necessary
if not os.path.isdir(os.path.dirname(outputfile)):
os.makedirs(os.path.dirname(outputfile))
# choose output file_type
if file_type == "root":
service = I3ROOTTableService(outputfile + '.root', 'master_tree')
if verbose:
print('Storing in ' + outputfile + '.root')
elif file_type in ["h5", "hd5", "hdf5"]:
service = I3HDFTableService(outputfile + '.hd5')
if verbose:
print('Storing in ' + outputfile + '.hd5')
else:
service = I3HDFTableService(outputfile + '.hd5')
if verbose:
print 'Using standard file_type: hd5.'
print('Storing in ' + outputfile + '.hd5')
if generateID:
tray.AddModule(generic_attributes.create_event_id, 'HeaderModifier')
# write chosen attributes
tray.AddModule(I3TableWriter,
'writer',
tableservice=[service],
BookEverything=True,
SubEventStreams=[sub_event_stream])
# close file
tray.AddModule('TrashCan', 'can')
tray.Execute()
tray.Finish()
#!/usr/bin/env python
from icecube import icetray, tableio
from icecube.hdfwriter import I3HDFTableService
from icecube.rootwriter import I3ROOTTableService
from icecube.tableio import I3TableTranscriber
outservice = I3HDFTableService('ldf_proton_1PeV.hdf')
inservice = I3ROOTTableService('ldf_proton_1PeV.root', 'r')
scribe = I3TableTranscriber(inservice, outservice)
scribe.Execute()
def SimIceTop(tray, name,
input,
output=None,
gcd=None,
log_level=2,
samples=1,
tanks=[],
x=0.0,
y=0.0,
r=0.0,
seed=0,
rng="gsl",
injector="normal",
response="g4",
new=True,
pecompress=2,
weighted=False,
calibrate=False,
trigger=False):
"""
Simulate IceTop response.
Input can be a list of CORSIKA files or a tuple of ``(str, kwargs)``,
which are passed to ``tray.AddService(str, "particle-injector", **kwargs)``
to create a custom particle injector. RandomServiceName should not be
specified, it is overridden internally.
The option "weighted" only takes effect if the input are CORSIKA files.
"""
import os
import re
from icecube import icetray, dataclasses, dataio
from icecube.sim_services.sim_utils.gcd_utils import get_time
from icecube.icetray import I3Units
custom_injector = False
if len(input) == 2 and isinstance(input[0], str) and isinstance(input[1], dict):
custom_injector = True
if gcd is None:
raise ValueError("Need to specify a GCD file")
if not os.path.exists(gcd):
raise IOError("Specified GCD file does not exist")
icetray.set_log_level_for_unit('I3TopSimulator', icetray.I3LogLevel(log_level))
icetray.set_log_level_for_unit('I3CorsikaReader', icetray.I3LogLevel(log_level))
icetray.set_log_level_for_unit('I3CorsikaInjector', icetray.I3LogLevel(log_level))
icetray.set_log_level_for_unit('I3G4TankResponse', icetray.I3LogLevel(log_level))
icetray.set_log_level_for_unit('I3ParamTankResponse', icetray.I3LogLevel(log_level))
# to be used by injector
for tag in ("inj", "resp", "other"):
if rng == "gsl":
tray.AddService("I3GSLRandomServiceFactory", "gslrandom_" + tag,
Seed=seed,
InstallServiceAs='random_' + tag)
elif rng == "root":
tray.AddService("I3TRandomServiceFactory", "rootrandom_" + tag,
Seed=seed,
InstallServiceAs='random_' + tag)
elif rng == "sprng":
tray.AddService("I3SPRNGRandomServiceFactory", "sprngrandom_" + tag,
Seed=seed,
NStreams=1,
StreamNum=0,
InstallServiceAs='random_' + tag)
else:
raise ValueError("Unknown randon number generator: " + rng)
if custom_injector:
tray.AddService(input[0], "particle-injector",
RandomServiceName = 'random_inj',
**input[1])
else:
# CORSIKA injector
if weighted:
if injector == "normal":
tray.AddService("I3InjectorFactory<I3CorsikaInjector>", "particle-injector",
FileNameList = input,
RandomServiceName = 'random_inj',
NumSamples = samples, # <-- Number of re-samples of the same shower
ImportanceSampling = True,
#PartDistr = os.path.join(os.path.dirname(options.output),
# 'on_regions_' + os.path.basename(options.output).replace(extension, 'root')),
Tanks = tanks,
IgnoreParticleTypes = [75, 76, 85, 86, 95, 96]
)
else:
raise ValueError("option weighted requires normal injector")
else:
if injector == "normal":
tray.AddService("I3InjectorFactory<I3CorsikaInjector>", "particle-injector",
FileNameList = input,
RandomServiceName = 'random_inj',
NumSamples = samples, # <-- Number of re-samples of the same shower
RelocationX = x, # <-- x-coordinate of core or resampling center (if Relocation R > 0)
RelocationY = y, # <-- y-coordinate or core or resampling center (if Relocation R > 0)
RelocationR = r, # <-- Re-sampling radius (if zero --> fixed core location)
Tanks = tanks,
IgnoreParticleTypes = [75, 76, 85, 86, 95, 96]
)
elif injector == "unthin":
tray.AddService("I3InjectorFactory<I3CorsikaThinnedInjector>", "particle-injector",
FileNameList = input,
RandomServiceName = 'random_inj',
NumSamples = samples, # <-- Number of re-samples of the same shower
RelocationX = x, # <-- x-coordinate of core or resampling center (if Relocation R > 0)
RelocationY = y, # <-- y-coordinate or core or resampling center (if Relocation R > 0)
RelocationR = r, # <-- Re-sampling radius (if zero --> fixed core location)
)
else:
raise ValueError("unknown injector option")
if response == 'g4':
from icecube import g4_tankresponse
tray.AddService("I3IceTopResponseFactory<I3G4TankResponse>", "topresponse",
RandomServiceName = "random_resp",
ChargeScale = 1.02
)
elif response == 'param':
tray.AddService("I3IceTopResponseFactory<I3ParamTankResponse>", "topresponse",
RandomServiceName = "random_resp",
UseSnowParam = True
)
else:
raise ValueError("Unknown IceTop tank response: " + response)
tray.AddModule("I3InfiniteSource", "source",
prefix = gcd,
stream = icetray.I3Frame.DAQ )
time = get_time(dataio.I3File(gcd))
tray.AddModule("I3MCEventHeaderGenerator","time-gen",
Year = time.utc_year,
DAQTime = time.utc_daq_time)
if new:
tray.AddSegment(SimulateNew, 'new_simulation',
InjectorServiceName = "particle-injector",
ResponseServiceName = "topresponse",
RandomServiceName = "random_other",
InIceMCTreeName = '',
Tanks = tanks,
CompressPEs=pecompress
)
else:
tray.AddSegment(SimulateOld, 'old_simulation',
InjectorServiceName = "particle-injector",
ResponseServiceName = "topresponse",
InIceMCTreeName = '',
Tanks = tanks
)
if calibrate:
from icecube import topsimulator
tray.AddSegment(CalibrateAndExtract, 'CalibrateExtract',
Launches = 'IceTopRawData',
)
tray.AddModule('I3TopAddComponentWaveforms', 'AddComponentWaveforms')
if output is not None: # writing of output is requested
if output.endswith(".h5") or output.endswith(".root"):
# write tables
keep_keys = ['I3EventHeader',
'MCTopCherenkovPulses',
'MCTopHitSeriesMap',
'IceTopPulses_HLC',
'IceTopPulses_SLC',
'IceTopHLCVEMPulses',
'IceTopSLCVEMPulses']
if custom_injector:
keep_keys += ['I3MCTreeIT',
'MCTopPulses']
else:
keep_keys += ['MCPrimary',
'MCPrimaryInfo',
'SamplingWeight',
'Samples']
tray.AddModule(DummySubEventSplit, 'split')
from icecube.tableio import I3TableWriter
if output.endswith('.h5'):
from icecube.hdfwriter import I3HDFTableService
hdf_service = I3HDFTableService(output)
tray.AddModule(I3TableWriter, "writer",
Keys = keep_keys,
TableService = [ hdf_service ],
SubEventStreams = ['IceTop'],
)
elif output.endswith('.root'):
from icecube.rootwriter import I3ROOTTableService
root_service = I3ROOTTableService(output)
tray.AddModule(I3TableWriter, "writer",
Keys = keep_keys,
TableService = [ root_service ],
SubEventStreams = ['IceTop'],
)
else:
# write standard output format i3
tray.AddModule("I3Writer", "i3-writer",
Filename = output,
streams = [icetray.I3Frame.DAQ]
)
def output_i3_root(i):
infile = file_list_all[i]
infile = [str(j) for j in infile]
print(infile,i)
#### NAME YOUR OUTPUT FILES HERE
# Put this someplace that exists! (This particular one is for KR)
I3_OUTFILE = HOME+'/'+data_set_number+'/'+'Subset_%s'%(i)+".i3"
ROOTFILE = HOME+'/'+data_set_number+'/'+'Subsets_%s'%(i)+".root"
tray = I3Tray()
########## SERVICES FOR GULLIVER ##########
datareadoutName="IceTopHLCSeedRTPulses"
badtanksName= "IceTopHLCSeedRTExcludedTanks"
## The "simple lambda" snowservice
#tray.AddService("I3SimpleSnowCorrectionServiceFactory","SimpleSnow21")(
# ("Lambda", 2.1)
# )
## This one is the standard one.
tray.AddService("I3GulliverMinuitFactory","Minuit")(
("MinuitPrintLevel",-2),
("FlatnessCheck",True),
("Algorithm","MIGRAD"),
("MaxIterations",50000),
("MinuitStrategy",2),
("Tolerance",0.1),
)
tray.AddService("I3CurvatureSeedServiceFactory","CurvSeed")(
("SeedTrackName", "Laputop"), # This is also the default
("A", 6e-4), # This comes from the original IT-26 gausspar function
("N",9.9832),
("D",63.5775)
)
tray.AddService("I3CurvatureParametrizationServiceFactory","CurvParam")(
("FreeA", True),
("MinA", 0.0),
("MaxA", 2e-3),
("StepsizeA", 1e-5)
)
tray.AddService("I3CurvatureParametrizationServiceFactory","CurvParam2")(
("FreeN",True),
("MinN",0),
("MaxN",200.0),
("StepsizeN",2.0)
)
tray.AddService("I3CurvatureParametrizationServiceFactory","CurvParam3")(
("FreeD",True),
("MinD",0),
("MaxD",500.0),
("StepsizeD",2.0)
)
tray.AddService("I3LaputopLikelihoodServiceFactory","ToprecLike2")(
("datareadout", datareadoutName),
("badtanks", badtanksName),
("ldf", ""), # do NOT do the LDF (charge) likelihood
("curvature","gaussparfree") # yes, do the CURVATURE likelihood
# ("SnowServiceName","SimpleSnow21")
)
#------------------- LET'S RUN SOME MODULES! ------------------
#**************************************************
# Reader and whatnot
#**************************************************
tray.AddModule("I3Reader","reader")(
("FileNameList", [GCDfile]+infile)
)
tray.AddModule("I3LaputopFitter","CurvatureOnly")(
("SeedService","CurvSeed"),
("NSteps",3), # <--- tells it how many services to look for and perform
("Parametrization1","CurvParam"),
("Parametrization2","CurvParam2"),
("Parametrization3","CurvParam3"),
("StoragePolicy","OnlyBestFit"),
("Minimizer","Minuit"),
("LogLikelihoodService","ToprecLike2"), # the three likelihoods
("LDFFunctions",["","",""]), # do NOT do the LDF (charge) likelihood
("CurvFunctions",["gaussparfree","gaussparfree","gaussparfree"]) # yes, do the CURVATURE likelihood
)
tray.AddModule("I3Writer","EventWriter")(
("DropOrphanStreams", [icetray.I3Frame.DAQ]),
("Filename",I3_OUTFILE),
)
## Output root file
root = I3ROOTTableService(ROOTFILE,"aTree")
tray.AddModule(I3TableWriter,'writer')(
("tableservice", root),
("keys",[ "Laputop",
"LaputopParams",
"CurvatureOnly",
"CurvatureOnlyParams",
"Millipede",
"MillipedeFitParams",
"Millipede_dEdX",
"Stoch_Reco",
"Stoch_Reco2",
"I3EventHeader",
"MCPrimary",
"MCPrimaryInfo",
"IT73AnalysisIceTopQualityCuts",
"IT73AnalysisInIceQualityCuts"]),
("subeventstreams",["ice_top"])
)
# Execute the Tray
# Just to make sure it's working!
tray.Execute()
if 'root' in [iformat, oformat]:
from icecube.rootwriter import I3ROOTTableService
if 'csv' in [iformat, oformat]:
from icecube.textwriter import I3CSVTableService
if iformat == 'hdf5':
inservices = [(I3HDFTableService, (infile, 1, 'r')) for infile in infiles]
elif iformat == 'root':
inservices = [(I3ROOTTableService, (infile, 'r')) for infile in infiles]
else:
raise "Unknown input format '%s'" % iformat
if oformat == 'hdf5':
outservice = I3HDFTableService(outfile, options.compress, 'w')
elif oformat == 'root':
outservice = I3ROOTTableService(outfile,
compression_level=options.compress)
elif oformat == 'csv':
outservice = I3CSVTableService(outfile)
else:
raise "Unknown out format '%s'" % oformat
for ctor, args in inservices:
print('Merging %s' % args[0])
inservice = ctor(*args)
scribe = I3TableTranscriber(inservice, outservice)
if options.nframes is not None:
scribe.Execute(options.nframes)
else:
scribe.Execute()
inservice.Finish()
def ExtractDST(tray, name,
dst_output_filename = "I3DST.hdf5",
dstname = "I3DST",
simulation = False,
extract_to_frame = False,
remove_filter_stream = True,
cut_data = False,
If = lambda f: True):
"""
Record in compact form limited information from reconstructions, triggers and cut
parameters for every triggered event.
"""
from icecube import dst
from icecube import phys_services
from icecube.tableio import I3TableWriter
from . import TDSTConverter
if dst_output_filename.endswith('.root'):
# Open output file
from icecube.rootwriter import I3ROOTTableService
table_service = I3ROOTTableService(filename= dst_output_filename,
master= "dst", #Default name: "MasterTree".
#mode=RECREATE,
)
elif dst_output_filename.endswith('.hdf5'):
from icecube.hdfwriter import I3HDFTableService
table_service = I3HDFTableService(dst_output_filename, 6)
if "I3RandomService" not in tray.context:
dstRng = phys_services.I3GSLRandomService(int(time.time()))
tray.context["I3RandomService"]=dstRng
if simulation:
from icecube import filter_tools
tray.AddModule("KeepFromSubstream","dst_stream",
StreamName = "InIceSplit",
KeepKeys = ["I3DST"],
KeepStream=True,
)
tray.AddModule('I3DSTExtractor16', 'UnpackDST',
SubEventStreamName = 'TDST',
FileName = dst_output_filename,
DSTName = dstname,
DSTHeaderName = "I3DSTHeader",
EventHeaderName = 'I3EventHeader',
ExtractToFrame = extract_to_frame,
TriggerName = 'I3TriggerHierarchy',
Cut = cut_data,
)
tray.AddModule(I3TableWriter, "writer",
TableService = table_service,
SubEventStreams= ['TDST'],
Keys = [ "CutDST", "TDSTTriggers"],
)
if remove_filter_stream:
tray.AddModule(filterStream,name+'_stream_filter', StreamName='TDST')
tray.Add("I3NullSplitter", "nullsplit")
tray.Add(fill_frame)
from icecube.tableio import I3BroadcastTableService
tablers = [tableio.I3CSVTableService('test_converters')]
outfiles = ['test_converters']
try:
from icecube.hdfwriter import I3HDFTableService
tablers.append(I3HDFTableService("test_converters.hdf5", 6, 'w'))
outfiles.append('test_converters.hdf5')
except ImportError:
pass
try:
from icecube.rootwriter import I3ROOTTableService
tablers.append(I3ROOTTableService("test_converters.root"))
outfiles.append('test_converters.root')
except ImportError:
pass
if len(tablers) == 1:
tabler = tablers[0]
else:
tabler = I3BroadcastTableService(tuple(tablers))
tray.Add(tableio.I3TableWriter,
TableService=tabler,
BookEverything=True,
SubEventStreams=["nullsplit"],
)
tray.Execute(1)
outfile = sys.argv[1]
infiles = sys.argv[2:]
#EXAMPLE HOW TO RUN IT: python $I3_SRC/rootwriter/resources/examples/book_data.py myoutput.root /data/sim/IceCube/2011/filtered/level2/CORSIKA-in-ice/10309/00000-00999/Level2_IC86.2011_corsika.010309.000000.i3.bz2
tray = I3Tray()
# Read .i3 file
tray.AddModule("I3Reader", "reader",
FileNameList = infiles)
# Open output file
table_service = I3ROOTTableService(filename= outfile, #Name of the output TFile which contains TTree.
master= "FullTree", #Default name: "MasterTree".
#mode=RECREATE, #RECREATE is the default one, more options (NEW,UPDATE,etc) in:
#https://root.cern.ch/root/html/TFile.html .
)
# Book data
tray.AddModule(I3TableWriter, "writer",
TableService = table_service,
SubEventStreams= ['in_ice'], #Book events happening InIce. One also can use SubEventStreams= ['in_ice','ice_top'].
Keys = [ "SPEFit2", "SPEFitSingle" ], #This list has to contain all the objects that you want to book.
#BookEverything=True, #Will book every thing in the frame, and your file will be very large,
#we suggest "do not do that". Default is False.
)
tray.AddModule("TrashCan", "trash")
tray.Execute()
def function2(i):
I3_OUTFILE = output_directory + data_set_number + '_%s_%s' % (
i[1], i[2]) + '.i3.bz2'
ROOTFILE = output_directory + data_set_number + '_%s_%s' % (i[1],
i[2]) + '.root'
tray = I3Tray()
########## SERVICES FOR GULLIVER ##########
#------------------- LET'S RUN SOME MODULES! ------------------
#**************************************************
# Reader and whatnot
#**************************************************
datareadoutName = 'IceTopLaputopSeededSelectedHLC'
badtanksName = "BadDomsList"
print(i[0])
tray.AddModule("I3Reader", "reader")(("FileNameList", i[0]))
tray.Add(uncompress)
# Extract HLC pulses
#tray.AddModule('I3TopHLCPulseExtractor', 'TopHLCPulseExtractor',
# PulseInfo = 'IceTopHLCPulseInfo', # PulseInfo: amplitude, rise time, etc. Empty string to disable
# Waveforms = 'IceTopVEMCalibratedHLCWaveforms', # Input HLC waveforms from WaveCalibrator
# )
tray.AddModule(Process_Waveforms, 'Process_wavefomrs')
tray.AddModule(Extract_info)
tray.AddModule(Get_data)
tray.AddSegment(LaputopStandard, "Laputop_new", pulses='LaputopHLCVEM')
tray.AddModule(Get_fit)
tray.AddModule("I3Writer", "EventWriter")(
("DropOrphanStreams", [icetray.I3Frame.DAQ]),
("Filename", I3_OUTFILE),
)
wanted_inice_reco = [
"Millipede", "MillipedeFitParams", "Millipede_dEdX", "Stoch_Reco",
"Stoch_Reco2", "I3MuonEnergyLaputopCascadeParams",
"I3MuonEnergyLaputopParams"
]
wanted_inice_cuts = ['IT73AnalysisInIceQualityCuts']
wanted_general = [
'I3EventHeader',
'CalibratedHLCWaveforms',
'CalibratedSLCWaveforms',
'LaputopHLCWaveforms',
'IceTopWaveformWeight',
'IceTopVEMCalibratedWaveforms',
'IceTopHLCPEPulses',
#'IceTopHLCPulseInfo',
'IceTopHLCVEMPulses',
'IceTopSLCPEPulses',
'IceTopSLCVEMPulses',
'LaputopHLCVEM',
'LaputopSLCVEM',
'Laputop',
'LaputopParams',
'Laputop_new',
'Laputop_newParams'
'All_pulses',
'All_radius',
'All_radius_old',
'ShowerCOG',
'm',
's',
'chi2',
'sigma_m',
'sigma_s'
]
## Output root file
root = I3ROOTTableService(ROOTFILE, "aTree")
tray.AddModule(I3TableWriter, 'writer')(
("tableservice", root),
("keys", wanted_general + wanted_inice_reco + wanted_inice_cuts),
("subeventstreams", ['InIceSplit', "ice_top"]))
# Execute the Tray
# Just to make sure it's working!
tray.Execute()