def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
cfg = yaml.load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
infile = cfg['infile_pattern'].format(**cfg)
infile = infile.replace(' ', '0')
tray = I3Tray()
tray.context['I3FileStager'] = dataio.get_stagers()
random_service, _, run_id = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'])
tray.context['I3RandomService'] = random_service
tray.Add('I3Reader', FilenameList=[cfg['gcd_pass2'], infile])
if run_number < cfg['det_pass2_keep_all_upto']:
cfg['det_keep_mc_hits'] = True
cfg['det_keep_propagated_mc_tree'] = True
cfg['det_keep_mc_pulses'] = True
tray.AddSegment(segments.DetectorSim,
"Detector5Sim",
RandomService='I3RandomService',
RunID=run_id,
GCDFile=cfg['gcd_pass2'],
KeepMCHits=cfg['det_keep_mc_hits'],
KeepPropagatedMCTree=cfg['det_keep_propagated_mc_tree'],
KeepMCPulses=cfg['det_keep_mc_pulses'],
SkipNoiseGenerator=cfg['det_skip_noise_generation'],
LowMem=cfg['det_low_mem'],
InputPESeriesMapName=MCPE_SERIES_MAP,
BeaconLaunches=cfg['det_add_beacon_launches'],
FilterTrigger=cfg['det_filter_trigger'])
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace(' ', '0')
print(outfile)
print(cfg['outfile_pattern'])
tray.AddModule("I3Writer",
"EventWriter",
filename=outfile,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation
])
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
if int(yaml.__version__[0]) < 5:
# backwards compatibility for yaml versions before version 5
cfg = yaml.load(stream)
else:
cfg = yaml.full_load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace(' ', '0')
click.echo('Run: {}'.format(run_number))
click.echo('ParticleType: {}'.format(cfg['particle_type']))
click.echo('Outfile: {}'.format(outfile))
click.echo('n_events_per_run: {}'.format(cfg['n_events_per_run']))
click.echo('smearing_angle: {}'.format(cfg['smearing_angle']))
click.echo('skymap_path: {}'.format(cfg['skymap_path']))
tray = I3Tray()
random_services, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=2)
tray.AddModule(
'I3InfiniteSource',
'source',
# Prefix=gcdfile,
Stream=icetray.I3Frame.DAQ)
tray.AddModule(ParticleFactory,
'make_particles',
particle_type=cfg['particle_type'],
map_filename=cfg['skymap_path'],
num_events=cfg['n_events_per_run'],
smearing_angle=cfg['smearing_angle'] * I3Units.deg,
random_state=cfg['seed'],
random_service=random_services[0])
tray.AddSegment(segments.PropagateMuons,
'propagate_muons',
RandomService=random_services[1],
InputMCTreeName='I3MCTree',
**cfg['muon_propagation_config'])
tray.AddModule('I3Writer',
'write',
Filename=outfile,
Streams=[icetray.I3Frame.DAQ,
icetray.I3Frame.Stream('M')])
tray.AddModule('TrashCan', 'trash')
tray.Execute()
tray.Finish()
del tray
def run_snowstorm_propagation(cfg, infile, outfile):
"""Run SnowStorm Propagation.
Adopted from:
https://code.icecube.wisc.edu/projects/icecube/browser/IceCube/
meta-projects/combo/stable/simprod-scripts/resources/scripts/
SnowSuite/3-Snowstorm.py
Parameters
----------
cfg : dict
Dictionary with configuration settings.
infile : str
Path to input file.
outfile : str
Path to output file.
"""
start_time = time.time()
# --------
# Settings
# --------
default_args = {
# required
'NumEventsPerModel': 100,
'DOMOversizeFactor': 1.,
'UseI3PropagatorService': True,
# optional
'UseGPUs': True,
'SummaryFile': 'summary_snowstorm.yaml',
'UseOnlyDeviceNumber': None,
'MCTreeName': 'I3MCTree',
'OutputMCTreeName': None,
'FlasherInfoVectName': None,
'FlasherPulseSeriesName': None,
'PhotonSeriesName': None,
'MCPESeriesName': "I3MCPESeriesMap",
'DisableTilt': False,
'UnWeightedPhotons': False,
'UnWeightedPhotonsScalingFactor': None,
'UseGeant4': False,
'ParticleHistory': True,
'ParticleHistoryGranularity': 1*icetray.I3Units.m,
'CrossoverEnergyEM': None,
'CrossoverEnergyHadron': None,
'UseCascadeExtension': True,
'StopDetectedPhotons': True,
'PhotonHistoryEntries': 0,
'DoNotParallelize': False,
'UnshadowedFraction': 1.0,
'WavelengthAcceptance': None,
'DOMRadius': 0.16510*icetray.I3Units.m,
'CableOrientation': None,
'OverrideApproximateNumberOfWorkItems': None,
'IgnoreSubdetectors': ["IceTop"],
'ExtraArgumentsToI3CLSimClientModule': dict(),
}
# overwrite default settings
default_args.update(cfg)
cfg = default_args
snowstorm_config = cfg['snowstorm_config']
if cfg['SummaryFile'] is not None:
cfg['SummaryFile'] = cfg['SummaryFile'].format(**cfg)
ice_model_location = \
os.path.expandvars(snowstorm_config["IceModelLocation"])
hole_ice_parameterization = \
os.path.expandvars(snowstorm_config["HoleIceParameterization"])
# set units to meter
cfg['ParticleHistoryGranularity'] *= icetray.I3Units.m
cfg['DOMRadius'] *= icetray.I3Units.m
# Print out most important settings
click.echo('\n---------------')
click.echo('Script Settigns')
click.echo('---------------')
click.echo('\tInput: {}'.format(infile))
click.echo('\tGCDFile: {}'.format(cfg['gcd']))
click.echo('\tOutput: {}'.format(outfile))
for key in ['DOMOversizeFactor', 'UseI3PropagatorService', 'UseGPUs',
'SummaryFile']:
click.echo('\t{}: {}'.format(key, cfg[key]))
click.echo('---------------\n')
# get random service
random_services, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=1,
use_gslrng=cfg['random_service_use_gslrng'])
random_service = random_services[0]
"""
Setup and run Snowstorm (aka MultiSim) by running a series of short
trays, each with a different ice model. This works by front-loading as much
of the expensive initialization (reading the GCD file, setting up
PROPOSAL/Geant4, etc) as possible, so that only the propagation kernel
needs to be recompiled for every tray.
"""
# instantiate baseline detector setup.
# this will help construct the baseline characteristics before applying
# the perturbers
print("Setting up detector... ", end="")
clsimParams = setupDetector(
GCDFile=cfg['gcd'],
SimulateFlashers=bool(cfg['FlasherInfoVectName'] or
cfg['FlasherPulseSeriesName']),
IceModelLocation=ice_model_location,
DisableTilt=cfg['DisableTilt'],
UnWeightedPhotons=cfg['UnWeightedPhotons'],
UnWeightedPhotonsScalingFactor=cfg['UnWeightedPhotonsScalingFactor'],
UseI3PropagatorService=cfg['UseI3PropagatorService'],
UseGeant4=cfg['UseGeant4'],
CrossoverEnergyEM=cfg['CrossoverEnergyEM'],
CrossoverEnergyHadron=cfg['CrossoverEnergyHadron'],
UseCascadeExtension=cfg['UseCascadeExtension'],
StopDetectedPhotons=cfg['StopDetectedPhotons'],
DOMOversizeFactor=cfg['DOMOversizeFactor'],
UnshadowedFraction=cfg['UnshadowedFraction'],
HoleIceParameterization=hole_ice_parameterization,
WavelengthAcceptance=cfg['WavelengthAcceptance'],
DOMRadius=cfg['DOMRadius'],
CableOrientation=cfg['CableOrientation'],
IgnoreSubdetectors=cfg['IgnoreSubdetectors'],
)
print("done")
print("Setting up OpenCLDevices... ", end="")
openCLDevices = configureOpenCLDevices(
UseGPUs=cfg['UseGPUs'],
UseCPUs=not cfg['UseGPUs'],
OverrideApproximateNumberOfWorkItems=cfg[
'OverrideApproximateNumberOfWorkItems'],
DoNotParallelize=cfg['DoNotParallelize'],
UseOnlyDeviceNumber=cfg['UseOnlyDeviceNumber'])
print("done")
# -------------------
# Setup perturbations
# -------------------
# create empty "perturber" object
perturber = Perturber()
# get perturbation_cfg dict to simplify calls
perturbation_cfg = snowstorm_config["Perturbations"]
# loop over all perturbations in the perturbation_cfg
print("Setting up perturbers... ")
for name, params in perturbation_cfg.items():
# catch special case of IceWavePlusModes
if name == "IceWavePlusModes":
if not params["apply"]:
continue
if params["type"] == "default":
print("-> adding {} of type {}".format(name, params["type"]))
perturber.add('IceWavePlusModes',
*icewave.get_default_perturbation())
continue
elif hasattr(snowstorm_perturbers, params["type"]):
print("-> adding {} of type {}".format(name, params["type"]))
get_perturber = getattr(snowstorm_perturbers, params["type"])
perturber.add('IceWavePlusModes',
*get_perturber(**params['settings']))
continue
else:
msg = "IceWavePlusModes of type '{}' are not implemented(yet)."
raise NotImplementedError(msg.format(params["type"]))
# all other cases
if params["type"] == "delta":
print("-> adding {} of type {}".format(name, params["type"]))
params = params["delta"]
perturber.add(name, all_parametrizations[name],
DeltaDistribution(params["x0"]))
elif params["type"] == "gauss":
print("-> adding {} of type {}".format(name, params["type"]))
params = params["gauss"]
# Caution: MultivariateNormal expect the covariance matrix as
# first argument, so we need to use sigma**2
perturber.add(name, all_parametrizations[name],
MultivariateNormal(
dataclasses.I3Matrix(np.diag(params["sigma"])**2),
params["mu"]))
elif params["type"] == "uniform":
print("-> adding {} of type {}".format(name, params["type"]))
params = params["uniform"]
perturber.add(name, all_parametrizations[name],
UniformDistribution(
[dataclasses.make_pair(*limits)
for limits in params["limits"]]))
else:
msg = "Perturbation '{}' of type '{}' not implemented."
raise NotImplementedError(msg.format(name, params["type"]))
print("done")
# Setting up some other things
gcdFrames = list(dataio.I3File(cfg['gcd']))
inputStream = dataio.I3FrameSequence([infile])
summary = dataclasses.I3MapStringDouble()
intermediateOutputFiles = []
# --------------
# Run PhotonProp
# --------------
# start a model counter
model_counter = 0
# Execute photon propagation
print("Executing photon propagation...", end="")
while inputStream.more():
# measure CLSimInit time
time_CLSimInit_start = time.time()
tray = I3Tray()
tray.context['I3RandomService'] = random_service
tray.context['I3SummaryService'] = summary
# make a mutable copy of the config dict
config = dict(clsimParams)
# populate the M frame with I3FrameObjects from clsimParams
model = icetray.I3Frame('M')
for k, v in config.items():
if isinstance(v, icetray.I3FrameObject):
model[k] = v
# apply perturbations in the order they were configured
perturber.perturb(random_service, model)
# check for items in the M-frame that were changed/added
# by the perturbers
for k in model.keys():
if k.startswith('Snowstorm'):
# keep all Snowstorm keys
continue
if k not in config:
msg = "\n {} was put in the M frame, but does not appear in "
msg += "the CLSim configuration dict"
raise KeyError(msg.format(k))
if config[k] != model[k]:
# if an items was changed, copy it back to clsimParams
config[k] = model[k]
else:
# remove unmodified items from the M frame
del model[k]
# add "persistent" I3Reader
tray.Add(FrameSequenceReader,
Sequence=itertools.chain(gcdFrames, [model], inputStream))
# inject an S frame if it doesn't exist
tray.Add(EnsureSFrame, Enable=len(intermediateOutputFiles) == 0)
# write pertubations to frame
def populate_s_frame(frame):
perturber.to_frame(frame)
tray.Add(populate_s_frame, Streams=[icetray.I3Frame.Stream('S')])
# Add Bumper to stop the tray after NumEventsPerModel Q-frames
tray.Add(Bumper, NumFrames=cfg['NumEventsPerModel'])
# initialize CLSim server and setup the propagators
server_location = tempfile.mkstemp(prefix='clsim-server-')[1]
address = 'ipc://'+server_location
converters = setupPropagators(
random_service, config,
UseGPUs=cfg['UseGPUs'],
UseCPUs=not cfg['UseGPUs'],
OverrideApproximateNumberOfWorkItems=cfg[
'OverrideApproximateNumberOfWorkItems'],
DoNotParallelize=cfg['DoNotParallelize'],
UseOnlyDeviceNumber=cfg['UseOnlyDeviceNumber']
)
server = clsim.I3CLSimServer(
address, clsim.I3CLSimStepToPhotonConverterSeries(converters))
# stash server instance in the context to keep it alive
tray.context['CLSimServer'] = server
# recycle StepGenerator to prevent repeated, expensive initialization
if 'StepGenerator' in cfg['ExtraArgumentsToI3CLSimClientModule']:
stepGenerator = \
cfg['ExtraArgumentsToI3CLSimClientModule']['StepGenerator']
stepGenerator.SetMediumProperties(config['MediumProperties'])
stepGenerator.SetWlenBias(config['WavelengthGenerationBias'])
# add CLSim server to tray
module_config = \
tray.Add(
I3CLSimMakePhotonsWithServer,
ServerAddress=address,
DetectorSettings=config,
MCTreeName=cfg['MCTreeName'],
OutputMCTreeName=cfg['OutputMCTreeName'],
FlasherInfoVectName=cfg['FlasherInfoVectName'],
FlasherPulseSeriesName=cfg['FlasherPulseSeriesName'],
PhotonSeriesName=cfg['PhotonSeriesName'],
MCPESeriesName=cfg['MCPESeriesName'],
RandomService=random_service,
ParticleHistory=cfg['ParticleHistory'],
ParticleHistoryGranularity=cfg['ParticleHistoryGranularity'],
ExtraArgumentsToI3CLSimClientModule=cfg[
'ExtraArgumentsToI3CLSimClientModule'],
)
# recycle StepGenerator to prevent repeated, expensive initialization
cfg['ExtraArgumentsToI3CLSimClientModule']['StepGenerator'] = \
module_config['StepGenerator']
# write to temporary output file
intermediateOutputFiles.append(
tempfile.mkstemp(suffix=(outfile.split("/"))[-1])[1])
tray.Add("I3Writer",
Filename=intermediateOutputFiles[-1],
DropOrphanStreams=[icetray.I3Frame.TrayInfo],
Streams=[icetray.I3Frame.TrayInfo,
icetray.I3Frame.Simulation,
icetray.I3Frame.Stream('M'),
icetray.I3Frame.Stream('m'),
icetray.I3Frame.DAQ,
icetray.I3Frame.Physics])
# gather statistics in the "I3SummaryService"
tray.Add(GatherStatistics)
# measure CLSimInit time
time_CLSimInit = time.time() - time_CLSimInit_start
summary["CLSimInitTime_{:03d}".format(model_counter)] = time_CLSimInit
if "TotalCLSimInitTime" not in summary:
summary["TotalCLSimInitTime"] = time_CLSimInit
else:
summary["TotalCLSimInitTime"] += time_CLSimInit
# measure CLSimTray time
time_CLSimTray_start = time.time()
# Execute Tray
tray.Execute()
# measure CLSimTray time
time_CLSimTray = time.time() - time_CLSimTray_start
summary["CLSimTrayTime_{:03d}".format(model_counter)] = time_CLSimTray
if "TotalCLSimTrayTime" not in summary:
summary["TotalCLSimTrayTime"] = time_CLSimTray
else:
summary["TotalCLSimTrayTime"] += time_CLSimTray
# remove the temp file made by the server location thingy
os.unlink(server_location)
# increase model counter
model_counter += 1
print("done")
# Add number of models to summary
summary["TotalNumberOfModels"] = model_counter
# Concatenate intermediate files
print("Concatenating temporary files... ", end='')
tray = I3Tray()
tray.Add(dataio.I3Reader, "I3Reader", FilenameList=intermediateOutputFiles)
tray.Add("I3Writer", Filename=outfile,
DropOrphanStreams=[icetray.I3Frame.TrayInfo],
Streams=[icetray.I3Frame.TrayInfo,
icetray.I3Frame.Simulation,
icetray.I3Frame.Stream('M'),
icetray.I3Frame.Stream('m'),
icetray.I3Frame.DAQ,
icetray.I3Frame.Physics])
tray.Execute()
tray.Finish()
print("done")
print("Cleaning up Temporary files... ")
for fname in intermediateOutputFiles:
os.unlink(fname)
print("done")
# Recalculate averages
print("Writing summary file... ", end='')
if cfg['UseGPUs']:
if summary['TotalHostTime'] > 0.0:
summary['DeviceUtilization'] = \
summary['TotalDeviceTime']/summary['TotalHostTime']
if summary['TotalNumPhotonsGenerated'] > 0.0:
summary['AverageDeviceTimePerPhoton'] = \
summary['TotalDeviceTime']/summary['TotalNumPhotonsGenerated']
if summary['TotalNumPhotonsGenerated'] > 0.0:
summary['AverageHostTimePerPhoton'] = \
summary['TotalHostTime']/summary['TotalNumPhotonsGenerated']
if cfg['SummaryFile']:
with open(cfg['SummaryFile'], 'w') as f:
yaml.dump(dict(summary), f)
print("done")
print('--------')
print('Summary:')
print('--------')
for key, value in summary.items():
print('\t{}: {}'.format(key, value))
print('--------\n')
# Hurray!
print("All finished!")
# say something about the runtime
end_time = time.time()
print("That took "+str(end_time - start_time)+" seconds.")
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
if int(yaml.__version__[0]) < 5:
# backwards compatibility for yaml versions before version 5
cfg = yaml.load(stream)
else:
cfg = yaml.full_load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
infile = cfg['infile_pattern'].format(**cfg)
infile = infile.replace(' ', '0')
infile = infile.replace('Level0.{}'.format(cfg['previous_step']),
'Level0.{}'.format(cfg['previous_step'] % 10))
infile = infile.replace('2012_pass2', 'pass2')
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace('Level0.{}'.format(cfg['step']),
'Level0.{}'.format(cfg['step'] % 10))
outfile = outfile.replace(' ', '0')
outfile = outfile.replace('2012_pass2', 'pass2')
print('Outfile != $FINAL_OUT clean up for crashed scripts not possible!')
# collect keys which are to be kept in a addition to the official keep keys
# of the standard L1 and L2 processing
if 'additional_keep_keys' in cfg:
additional_keep_keys = cfg['additional_keep_keys'] + muongun_keys
else:
additional_keep_keys = muongun_keys
additional_keep_keys += [
'BiasedMuonWeighter', 'BiasedMuonCorridorWeighter',
'BiasedMESCHotspotWeighter', 'BiasedSimulationWeight',
'PROPOSALStorm', 'PROPOSALStormUniformRanges',
'MCVetoMuonInjectionInfo', 'MMCTrackListVetoMuon',
'CombinedMuonVetoI3MCTree', 'I3MCTreeVetoMuon',
'I3MCTreeVetoMuon_preMuonProp',
'I3MCTreeVetoMuon_preMuonProp_RNGState',
]
tray = I3Tray()
"""The main L1 script"""
tray.AddModule('I3Reader',
'i3 reader',
FilenameList=[cfg['gcd_pass2'], infile])
# run online filters
online_kwargs = {}
if SPLINE_TABLES:
online_kwargs.update({
'SplineRecoAmplitudeTable': os.path.join(
SPLINE_TABLES, 'InfBareMu_mie_abs_z20a10.fits'),
'SplineRecoTimingTable': os.path.join(
SPLINE_TABLES, 'InfBareMu_mie_prob_z20a10.fits'),
# 'alert_followup_base_GCD_filename': cfg['gcd_pass2'],
})
if cfg['L1_pass2_run_gfu'] is not None:
online_kwargs['gfu_enabled'] = cfg['L1_pass2_run_gfu']
if 'L1_needs_wavedeform_spe_corr' not in cfg:
cfg['L1_needs_wavedeform_spe_corr'] = False
tray.AddSegment(OnlineFilter, "OnlineFilter",
decode=False, simulation=True,
vemcal_enabled=False,
alert_followup=False,
needs_wavedeform_spe_corr=cfg[
'L1_needs_wavedeform_spe_corr'],
**online_kwargs
)
# make random service
_, seed = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=0)
filter_mask_randoms = phys_services.I3GSLRandomService(seed)
# override MinBias Prescale
filterconfigs = filter_globals.filter_pairs + filter_globals.sdst_pairs
print(cfg['L1_min_bias_prescale'])
if cfg['L1_min_bias_prescale']:
for i, filtertuple in enumerate(filterconfigs):
if filtertuple[0] == filter_globals.FilterMinBias:
del filterconfigs[i]
filterconfigs.append((filtertuple[0],
cfg['L1_min_bias_prescale']))
break
print(filterconfigs)
# Generate filter Masks for all P frames
tray.AddModule(filter_tools.FilterMaskMaker,
"MakeFilterMasks",
OutputMaskName=filter_globals.filter_mask,
FilterConfigs=filterconfigs,
RandomService=filter_mask_randoms)
# Merge the FilterMasks
tray.AddModule("OrPframeFilterMasks",
"make_q_filtermask",
InputName=filter_globals.filter_mask,
OutputName=filter_globals.qfilter_mask)
# Q+P frame specific keep module needs to go first, as KeepFromSubstram
# will rename things, let's rename post keep.
def is_Q(frame):
return frame.Stop == frame.DAQ
simulation_keeps = [
'BackgroundI3MCTree',
'BackgroundI3MCTreePEcounts',
'BackgroundI3MCPESeriesMap',
'BackgroundI3MCTree_preMuonProp',
'BackgroundI3MCTree_preMuonProp_RNGState',
'BackgroundMMCTrackList',
'BeaconLaunches',
'CorsikaInteractionHeight',
'CorsikaWeightMap',
'EventProperties',
'GenerationSpec',
'I3LinearizedMCTree',
'I3MCTree',
'I3MCTreePEcounts',
'I3MCTree_preMuonProp',
'I3MCTree_preMuonProp_RNGState',
'I3MCPESeriesMap',
'I3MCPESeriesMapWithoutNoise',
'I3MCPESeriesMapParticleIDMap',
'I3MCPulseSeriesMap',
'I3MCPulseSeriesMapParticleIDMap',
'I3MCPulseSeriesMapPrimaryIDMap',
'I3MCWeightDict',
'LeptonInjectorProperties',
'MCHitSeriesMap',
'MCPrimary',
'MCPrimaryInfo',
'MMCTrackList',
'PolyplopiaInfo',
'PolyplopiaPrimary',
'RNGState',
'SignalI3MCPEs',
'SimTrimmer', # for SimTrimmer flag
'TimeShift', # the time shift amount
'WIMP_params', # Wimp-sim
'noise_weight', # weights for noise-only vuvuzela simulations
'I3GENIEResultDict' # weight informaition for GENIE simulations
] + additional_keep_keys
keep_before_merge = filter_globals.q_frame_keeps + [
'InIceDSTPulses', # keep DST pulse masks
'IceTopDSTPulses',
'CalibratedWaveformRange', # keep calibration info
'UncleanedInIcePulsesTimeRange',
'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange',
'CalibrationErrata',
'SaturationWindows',
'InIceRawData', # keep raw data for now
'IceTopRawData',
] + simulation_keeps
tray.AddModule("Keep", "keep_before_merge",
keys=keep_before_merge,
If=is_Q)
# second set of prekeeps, conditional on filter content, based on newly
# created Qfiltermask
# Determine if we should apply harsh keep for events that failed to pass
# any filter
# Note: excluding the sdst_streams entries
tray.AddModule("I3IcePickModule<FilterMaskFilter>", "filterMaskCheckAll",
FilterNameList=filter_globals.filter_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedAnyFilter",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def do_save_just_superdst(frame):
if frame.Has("PassedAnyFilter"):
if not frame["PassedAnyFilter"].value:
return True # <- Event failed to pass any filter.
else:
return False # <- Event passed some filter
else:
icetray.logging.log_error("Failed to find key frame Bool!!")
return False
keep_only_superdsts = filter_globals.keep_nofilterpass + [
'PassedAnyFilter',
'InIceDSTPulses',
'IceTopDSTPulses',
'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange',
'RNGState'] + simulation_keeps
tray.AddModule("Keep", "KeepOnlySuperDSTs",
keys=keep_only_superdsts,
If=do_save_just_superdst)
# Now clean up the events that not even the SuperDST filters passed on
tray.AddModule("I3IcePickModule<FilterMaskFilter>", "filterMaskCheckSDST",
FilterNameList=filter_globals.sdst_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedKeepSuperDSTOnly",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def dont_save_superdst(frame):
if frame.Has("PassedKeepSuperDSTOnly") and \
frame.Has("PassedAnyFilter"):
if frame["PassedAnyFilter"].value:
return False # <- these passed a regular filter, keeper
elif not frame["PassedKeepSuperDSTOnly"].value:
return True # <- Event failed to pass SDST filter.
else:
return False # <- Event passed some SDST filter
else:
icetray.logging.log_error("Failed to find key frame Bool!!")
return False
# backward compatibility
if 'L1_keep_untriggered' in cfg and cfg['L1_keep_untriggered']:
discard_substream_and_keys = False
else:
discard_substream_and_keys = True
if discard_substream_and_keys:
tray.AddModule(
"Keep", "KeepOnlyDSTs",
keys=filter_globals.keep_dst_only + [
"PassedAnyFilter",
"PassedKeepSuperDSTOnly",
filter_globals.eventheader] + additional_keep_keys,
If=dont_save_superdst)
# Frames should now contain only what is needed. now flatten,
# write/send to server
# Squish P frames back to single Q frame, one for each split:
tray.AddModule("KeepFromSubstream", "null_stream",
StreamName=filter_globals.NullSplitter,
KeepKeys=filter_globals.null_split_keeps)
in_ice_keeps = filter_globals.inice_split_keeps + \
filter_globals.onlinel2filter_keeps
in_ice_keeps = in_ice_keeps + [
'I3EventHeader',
'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'TriggerSplitterLaunchWindow',
'I3TriggerHierarchy',
'GCFilter_GCFilterMJD'] + additional_keep_keys
tray.AddModule("Keep", "inice_keeps",
keys=in_ice_keeps,
If=which_split(split_name=filter_globals.InIceSplitter),)
tray.AddModule("KeepFromSubstream", "icetop_split_stream",
StreamName=filter_globals.IceTopSplitter,
KeepKeys=filter_globals.icetop_split_keeps)
# Apply small keep list (SuperDST/SmallTrig/DST/FilterMask for non-filter
# passers
# Remove I3DAQData object for events not passing one of the
# 'filters_keeping_allraw'
tray.AddModule("I3IcePickModule<FilterMaskFilter>", "filterMaskCheck",
FilterNameList=filter_globals.filters_keeping_allraw,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedConventional",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
# Clean out the Raw Data when not passing conventional filter
def I3RawDataCleaner(frame):
if not (('PassedConventional' in frame and
frame['PassedConventional'].value == True) or
('SimTrimmer' in frame and
frame['SimTrimmer'].value == True)):
frame.Delete('InIceRawData')
frame.Delete('IceTopRawData')
tray.AddModule(I3RawDataCleaner,
"CleanErrataForConventional",
Streams=[icetray.I3Frame.DAQ])
tray.AddModule("I3Writer", "EventWriter",
filename=outfile,
Streams=[icetray.I3Frame.DAQ,
icetray.I3Frame.Physics,
icetray.I3Frame.TrayInfo,
icetray.I3Frame.Simulation,
icetray.I3Frame.Stream('m'),
icetray.I3Frame.Stream('M')])
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
cfg = yaml.full_load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace(' ', '0')
click.echo('Run: {}'.format(run_number))
click.echo('Outfile: {}'.format(outfile))
for setting_key in (
'GenerateCosmicRayMuonsSettings',
'MuonGeometryFilterSettings',
'MuonLossProfileFilterSettings',
):
if cfg[setting_key]:
click.echo('{}:'.format(setting_key))
for setting, value in cfg[setting_key].items():
click.echo('\t{}: {}'.format(setting, value))
# crate random services
if 'random_service_use_gslrng' not in cfg:
cfg['random_service_use_gslrng'] = False
random_services, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=2,
use_gslrng=cfg['random_service_use_gslrng'])
# --------------------------------------
# Build IceTray
# --------------------------------------
tray = I3Tray()
# add random generator to tray context
tray.context['I3RandomService'] = random_services[0]
tray.AddModule('I3InfiniteSource',
'source',
Prefix=cfg['gcd'],
Stream=icetray.I3Frame.DAQ)
if 'oversampling_factor' not in cfg:
cfg['oversampling_factor'] = None
if 'oversample_after_proposal' in cfg and \
cfg['oversample_after_proposal']:
oversampling_factor_injection = None
oversampling_factor_photon = cfg['oversampling_factor']
else:
oversampling_factor_injection = cfg['oversampling_factor']
oversampling_factor_photon = None
tray.AddSegment(segments.GenerateCosmicRayMuons,
'GenerateCosmicRayMuons',
num_events=cfg['n_events_per_run'],
**cfg['GenerateCosmicRayMuonsSettings'])
# add filter to bias simulation based on track geometry
tray.AddModule(MuonGeometryFilter, 'MuonGeometryFilter',
**cfg['MuonGeometryFilterSettings'])
# add corridor MuonGun bias module:
# Events can be biased while keeping ability to properly weight events.
# This is not the case for the filter module above.
bias_mesc_hotspot_muons(tray, cfg)
bias_corridor_muons(tray, cfg)
tray.AddModule(DAQFrameMultiplier,
'PreDAQFrameMultiplier',
oversampling_factor=oversampling_factor_injection,
mctree_keys=['I3MCTree_preMuonProp'])
# propagate muons if config exists in config
# Note: Snowstorm may perform muon propagation internally
if 'muon_propagation_config' in cfg:
tray.AddSegment(segments.PropagateMuons,
'propagate_muons',
RandomService=random_services[1],
**cfg['muon_propagation_config'])
else:
# In this case we are not propagating the I3MCTree yet, but
# are letting this be done by snowstorm propagation
# We need to add a key named 'I3MCTree', since snowstorm expects this
# It will propagate the particles for us.
tray.AddModule('DummyMCTreeRenaming', 'DummyMCTreeRenaming')
# add filter to bias simulation based on muon loss profile
tray.AddModule(MuonLossProfileFilter, 'MuonLossProfileFilter',
**cfg['MuonLossProfileFilterSettings'])
# add MuonGun bias module:
# Events can be biased while keeping ability to properly weight events.
# This is not the case for the filter modules above.
bias_muongun_events(tray, cfg)
tray.AddModule(DAQFrameMultiplier,
'PostDAQFrameMultiplier',
oversampling_factor=oversampling_factor_photon,
mctree_keys=['I3MCTree'])
# --------------------------------------
# Distance Splits
# --------------------------------------
if cfg['distance_splits'] is not None:
click.echo('SplittingDistance: {}'.format(cfg['distance_splits']))
distance_splits = np.atleast_1d(cfg['distance_splits'])
dom_limits = np.atleast_1d(cfg['threshold_doms'])
if len(dom_limits) == 1:
dom_limits = np.ones_like(distance_splits) * cfg['threshold_doms']
oversize_factors = np.atleast_1d(cfg['oversize_factors'])
order = np.argsort(distance_splits)
distance_splits = distance_splits[order]
dom_limits = dom_limits[order]
oversize_factors = oversize_factors[order]
stream_objects = generate_stream_object(distance_splits, dom_limits,
oversize_factors)
tray.AddModule(OversizeSplitterNSplits,
"OversizeSplitterNSplits",
thresholds=distance_splits,
thresholds_doms=dom_limits,
oversize_factors=oversize_factors)
for stream_i in stream_objects:
outfile_i = stream_i.transform_filepath(outfile)
tray.AddModule("I3Writer",
"writer_{}".format(stream_i.stream_name),
Filename=outfile_i,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
],
If=stream_i)
click.echo('Output ({}): {}'.format(stream_i.stream_name,
outfile_i))
else:
click.echo('Output: {}'.format(outfile))
tray.AddModule("I3Writer",
"writer",
Filename=outfile,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
])
# --------------------------------------
click.echo('Scratch: {}'.format(scratch))
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
cfg = yaml.full_load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace(' ', '0')
infile = cfg['infile_pattern'].format(**cfg)
infile = infile.replace(' ', '0')
click.echo('Run: {}'.format(run_number))
click.echo('Outfile: {}'.format(outfile))
# -----------------------------
# Set PPC environment variables
# -----------------------------
ppc_config = cfg['ppc_config']
# define default Environment variables
ice_base = '$I3_BUILD/ice-models/resources/models/'
default_ppc_environment_variables = {
'PPCTABLESDIR': ice_base + 'spice_bfr-dv1_complete',
'OGPU': '1', # makes sure only GPUs are used (with OpenCL version)
}
ppc_environment_variables = dict(default_ppc_environment_variables)
ppc_environment_variables.update(ppc_config['environment_variables'])
# define default PPC arguments
default_ppc_arguments = {
'MCTree': 'I3MCTree',
}
if 'CUDA_VISIBLE_DEVICES' in os.environ:
default_ppc_arguments['gpu'] = int(os.environ['CUDA_VISIBLE_DEVICES'])
ppc_arguments = dict(default_ppc_arguments)
ppc_arguments.update(ppc_config['arguments'])
click.echo('PPC Settings:')
for key, value in ppc_environment_variables.items():
click.echo('\t{}: {}'.format(key, os.path.expandvars(value)))
os.putenv(key, os.path.expandvars(value))
click.echo('PPC Arguments:')
for key, value in ppc_arguments.items():
click.echo('\t{}: {}'.format(key, value))
# importing ppc must be done *after* setting the environment variables
from icecube import ppc
# ------------------------------
# get random service
random_services, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=1,
use_gslrng=cfg['random_service_use_gslrng'])
random_service = random_services[0]
# --------------------------------------
# Build IceTray
# --------------------------------------
tray = I3Tray()
tray.AddModule('I3Reader',
'i3 reader',
FilenameList=[cfg['gcd_pass2'], infile])
# run PPC
tray.context["I3RandomService"] = random_service
tray.AddModule("i3ppc", 'ppc', **ppc_arguments)
# rename MCPESeriesMap to I3MCPESeriesMap
tray.Add("Rename", keys=["MCPESeriesMap", "I3MCPESeriesMap"])
click.echo('Output: {}'.format(outfile))
tray.AddModule("I3Writer",
"writer",
Filename=outfile,
Streams=[
icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation,
icetray.I3Frame.Stream('M'),
icetray.I3Frame.Stream('S'), icetray.I3Frame.DAQ,
icetray.I3Frame.Physics
])
# --------------------------------------
click.echo('Scratch: {}'.format(scratch))
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def process_single_stream(cfg, infile, outfile):
click.echo('Input: {}'.format(infile))
hybrid_mode = (cfg['clsim_hybrid_mode']
and cfg['icemodel'].lower() != 'spicelea')
ignore_muon_light = (cfg['clsim_ignore_muon_light']
and cfg['clsim_hybrid_mode'])
click.echo('UseGPUs: {}'.format(cfg['clsim_usegpus']))
click.echo('IceModel: {}'.format(cfg['icemodel']))
if not cfg['icemodel_location'] is None:
click.echo('IceModelLocation: {}'.format(cfg['icemodel_location']))
click.echo('DomOversize {}'.format(cfg['clsim_dom_oversize']))
click.echo('UnshadowedFraction: {0:.2f}'.format(
cfg['clsim_unshadowed_fraction']))
click.echo('HybridMode: {}'.format(hybrid_mode))
click.echo('IgnoreMuonLight: {}'.format(ignore_muon_light))
click.echo('KeepMCPE: {}'.format(cfg['clsim_keep_mcpe']))
click.echo('Output: {}'.format(outfile))
tray = I3Tray()
tray.context['I3FileStager'] = dataio.get_stagers()
random_services, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=process_single_stream.n_streams)
random_service = random_services[process_single_stream.i_th_stream]
tray.context['I3RandomService'] = random_service
tray.Add('I3Reader', FilenameList=[cfg['gcd'], infile])
if hybrid_mode:
cascade_tables = segments.LoadCascadeTables(IceModel=cfg['icemodel'],
TablePath=SPLINE_TABLES)
else:
cascade_tables = None
if cfg['clsim_usegpus']:
use_gpus = True
use_cpus = False
else:
use_gpus = True
use_cpus = False
tray.AddSegment(segments.PropagatePhotons,
"PropagatePhotons",
RandomService=random_service,
MaxParallelEvents=MAX_PARALLEL_EVENTS,
KeepIndividualMaps=cfg['clsim_keep_mcpe'],
IceModel=cfg['icemodel'],
IceModelLocation=cfg['icemodel_location'],
UnshadowedFraction=cfg['clsim_unshadowed_fraction'],
IgnoreMuons=ignore_muon_light,
HybridMode=hybrid_mode,
UseGPUs=use_gpus,
UseAllCPUCores=use_cpus,
DOMOversizeFactor=cfg['clsim_dom_oversize'],
CascadeService=cascade_tables)
outfile = outfile.replace(' ', '0')
tray.AddModule("I3Writer",
"writer",
Filename=outfile,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
])
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
cfg = yaml.load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
if cfg['distance_splits'] is not None:
click.echo('SplittingDistances: {}'.format(cfg['distance_splits']))
click.echo('Oversizefactors: {}'.format(cfg['oversize_factors']))
click.echo('NEvents: {}'.format(cfg['n_events_per_run']))
click.echo('EMin: {}'.format(cfg['e_min']))
click.echo('EMax: {}'.format(cfg['e_max']))
click.echo('EBreak: {}'.format(cfg['muongun_e_break']))
click.echo('Gamma: {}'.format(cfg['gamma']))
click.echo('ZenithMin: {}'.format(cfg['zenith_min']))
click.echo('ZenithMax: {}'.format(cfg['zenith_max']))
tray = I3Tray()
random_service, random_service_prop, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'])
tray.context['I3RandomService'] = random_service
tray.AddModule("I3InfiniteSource",
"TheSource",
Prefix=cfg['gcd'],
Stream=icetray.I3Frame.DAQ)
tray.AddSegment(segments.GenerateSingleMuons,
"GenerateCosmicRayMuons",
NumEvents=cfg['n_events_per_run'],
FromEnergy=cfg['e_min'] * icetray.I3Units.GeV,
ToEnergy=cfg['e_max'] * icetray.I3Units.GeV,
BreakEnergy=cfg['muongun_e_break'] * icetray.I3Units.GeV,
GammaIndex=cfg['gamma'],
ZenithRange=[
cfg['zenith_min'] * icetray.I3Units.deg,
cfg['zenith_max'] * icetray.I3Units.deg
])
tray.AddSegment(segments.PropagateMuons,
"PropagateMuons",
RandomService=random_service_prop)
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace(' ', '0')
if cfg['distance_splits'] is not None:
click.echo('SplittingDistance: {}'.format(cfg['distance_splits']))
distance_splits = np.atleast_1d(cfg['distance_splits'])
dom_limits = np.atleast_1d(cfg['threshold_doms'])
if len(dom_limits) == 1:
dom_limits = np.ones_like(distance_splits) * cfg['threshold_doms']
oversize_factors = np.atleast_1d(cfg['oversize_factors'])
order = np.argsort(distance_splits)
distance_splits = distance_splits[order]
dom_limits = dom_limits[order]
oversize_factors = oversize_factors[order]
stream_objects = generate_stream_object(distance_splits, dom_limits,
oversize_factors)
tray.AddModule(OversizeSplitterNSplits,
"OversizeSplitterNSplits",
thresholds=distance_splits,
thresholds_doms=dom_limits,
oversize_factors=oversize_factors)
for stream_i in stream_objects:
outfile_i = stream_i.transform_filepath(outfile)
tray.AddModule("I3Writer",
"writer_{}".format(stream_i.stream_name),
Filename=outfile_i,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
],
If=stream_i)
click.echo('Output ({}): {}'.format(stream_i.stream_name,
outfile_i))
else:
click.echo('Output: {}'.format(outfile))
tray.AddModule("I3Writer",
"writer",
Filename=outfile,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
])
click.echo('Scratch: {}'.format(scratch))
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
cfg = yaml.full_load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace(' ', '0')
# ------------------------------
# get list of files for this run
# ------------------------------
import_cfg = cfg['event_import_settings']
glob_files = import_cfg['input_file_glob_list']
if isinstance(glob_files, str):
# single string provided
files = glob.glob(glob_files.format(run_number=run_number))
else:
# list of file globs provided
files = []
for file_pattern in glob_files:
files.extend(glob.glob(file_pattern.format(run_number=run_number)))
# sort files
files = sorted(files)
# ------------------------------
click.echo('Run: {}'.format(run_number))
click.echo('Outfile: {}'.format(outfile))
click.echo('Keys to import: {}'.format(import_cfg['keys_to_import']))
click.echo('input Files:')
for file in files:
click.echo('\t{}'.format(file))
# crate random services
if 'random_service_use_gslrng' not in cfg:
cfg['random_service_use_gslrng'] = False
random_services, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=2,
use_gslrng=cfg['random_service_use_gslrng'])
# --------------------------------------
# Build IceTray
# --------------------------------------
tray = I3Tray()
# import events from another I3-file
if 'num_events' not in import_cfg:
import_cfg['num_events'] = None
tray.AddModule(
ImportEvents,
'ImportEvents',
files=files,
num_events=import_cfg['num_events'],
keys_to_import=import_cfg['keys_to_import'],
rename_dict=import_cfg['rename_dict'],
mctree_name=import_cfg['mctree_name'],
)
# propagate muons if config exists in config
# Note: Snowstorm may perform muon propagation internally
if 'muon_propagation_config' in cfg:
tray.AddSegment(segments.PropagateMuons,
'propagate_muons',
RandomService=random_services[1],
**cfg['muon_propagation_config'])
tray.AddModule(DAQFrameMultiplier,
'DAQFrameMultiplier',
oversampling_factor=cfg['oversampling_factor'],
mctree_keys=[import_cfg['mctree_name']])
# --------------------------------------
# Distance Splits
# --------------------------------------
if cfg['distance_splits'] is not None:
click.echo('SplittingDistance: {}'.format(cfg['distance_splits']))
distance_splits = np.atleast_1d(cfg['distance_splits'])
dom_limits = np.atleast_1d(cfg['threshold_doms'])
if len(dom_limits) == 1:
dom_limits = np.ones_like(distance_splits) * cfg['threshold_doms']
oversize_factors = np.atleast_1d(cfg['oversize_factors'])
order = np.argsort(distance_splits)
distance_splits = distance_splits[order]
dom_limits = dom_limits[order]
oversize_factors = oversize_factors[order]
stream_objects = generate_stream_object(distance_splits, dom_limits,
oversize_factors)
tray.AddModule(OversizeSplitterNSplits,
"OversizeSplitterNSplits",
thresholds=distance_splits,
thresholds_doms=dom_limits,
oversize_factors=oversize_factors)
for stream_i in stream_objects:
outfile_i = stream_i.transform_filepath(outfile)
tray.AddModule("I3Writer",
"writer_{}".format(stream_i.stream_name),
Filename=outfile_i,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
],
If=stream_i)
click.echo('Output ({}): {}'.format(stream_i.stream_name,
outfile_i))
else:
click.echo('Output: {}'.format(outfile))
tray.AddModule("I3Writer",
"writer",
Filename=outfile,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
])
# --------------------------------------
click.echo('Scratch: {}'.format(scratch))
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
cfg = yaml.full_load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace(' ', '0')
click.echo('Run: {}'.format(run_number))
click.echo('Outfile: {}'.format(outfile))
click.echo('Azimuth: [{},{}]'.format(*cfg['azimuth_range']))
click.echo('Zenith: [{},{}]'.format(*cfg['zenith_range']))
click.echo('Energy: [{},{}]'.format(*cfg['primary_energy_range']))
click.echo('Vertex x: [{},{}]'.format(*cfg['x_range']))
click.echo('Vertex y: [{},{}]'.format(*cfg['y_range']))
click.echo('Vertex z: [{},{}]'.format(*cfg['z_range']))
# crate random services
if 'random_service_use_gslrng' not in cfg:
cfg['random_service_use_gslrng'] = False
random_services, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=2,
use_gslrng=cfg['random_service_use_gslrng'])
# --------------------------------------
# Build IceTray
# --------------------------------------
tray = I3Tray()
tray.AddModule(
'I3InfiniteSource',
'source',
# Prefix=gcdfile,
Stream=icetray.I3Frame.DAQ)
if 'max_vertex_distance' not in cfg:
cfg['max_vertex_distance'] = None
if 'constant_vars' not in cfg:
cfg['constant_vars'] = None
if 'sample_uniformly_on_sphere' not in cfg:
cfg['sample_uniformly_on_sphere'] = False
if 'oversample_after_proposal' in cfg and \
cfg['oversample_after_proposal']:
oversampling_factor_injection = None
oversampling_factor_photon = cfg['oversampling_factor']
else:
oversampling_factor_injection = cfg['oversampling_factor']
oversampling_factor_photon = None
tray.AddModule(
CascadeFactory,
'make_cascades',
azimuth_range=cfg['azimuth_range'],
zenith_range=cfg['zenith_range'],
sample_uniformly_on_sphere=cfg['sample_uniformly_on_sphere'],
primary_energy_range=cfg['primary_energy_range'],
fractional_energy_in_hadrons_range=cfg[
'fractional_energy_in_hadrons_range'],
time_range=cfg['time_range'],
x_range=cfg['x_range'],
y_range=cfg['y_range'],
z_range=cfg['z_range'],
max_vertex_distance=cfg['max_vertex_distance'],
flavors=cfg['flavors'],
interaction_types=cfg['interaction_types'],
num_events=cfg['n_events_per_run'],
oversampling_factor=oversampling_factor_injection,
random_state=cfg['seed'],
random_service=random_services[0],
constant_vars=cfg['constant_vars'],
)
# propagate muons if config exists in config
# Note: Snowstorm may perform muon propagation internally
if 'muon_propagation_config' in cfg:
tray.AddSegment(segments.PropagateMuons,
'propagate_muons',
RandomService=random_services[1],
**cfg['muon_propagation_config'])
else:
# In this case we are not propagating the I3MCTree yet, but
# are letting this be done by snowstorm propagation
# We need to add a key named 'I3MCTree', since snowstorm expects this
# It will propagate the particles for us.
tray.AddModule(DummyMCTreeRenaming, 'DummyMCTreeRenaming')
tray.AddModule(DAQFrameMultiplier,
'DAQFrameMultiplier',
oversampling_factor=oversampling_factor_photon)
# --------------------------------------
# Distance Splits
# --------------------------------------
if cfg['distance_splits'] is not None:
click.echo('SplittingDistance: {}'.format(cfg['distance_splits']))
distance_splits = np.atleast_1d(cfg['distance_splits'])
dom_limits = np.atleast_1d(cfg['threshold_doms'])
if len(dom_limits) == 1:
dom_limits = np.ones_like(distance_splits) * cfg['threshold_doms']
oversize_factors = np.atleast_1d(cfg['oversize_factors'])
order = np.argsort(distance_splits)
distance_splits = distance_splits[order]
dom_limits = dom_limits[order]
oversize_factors = oversize_factors[order]
stream_objects = generate_stream_object(distance_splits, dom_limits,
oversize_factors)
tray.AddModule(OversizeSplitterNSplits,
"OversizeSplitterNSplits",
thresholds=distance_splits,
thresholds_doms=dom_limits,
oversize_factors=oversize_factors)
for stream_i in stream_objects:
outfile_i = stream_i.transform_filepath(outfile)
tray.AddModule("I3Writer",
"writer_{}".format(stream_i.stream_name),
Filename=outfile_i,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
],
If=stream_i)
click.echo('Output ({}): {}'.format(stream_i.stream_name,
outfile_i))
else:
click.echo('Output: {}'.format(outfile))
tray.AddModule("I3Writer",
"writer",
Filename=outfile,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
])
# --------------------------------------
click.echo('Scratch: {}'.format(scratch))
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
cfg = yaml.full_load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace(' ', '0')
# ------------------------------
# get list of files for this run
# ------------------------------
import_cfg = cfg['event_import_settings']
glob_files = import_cfg['input_file_glob_list']
import_cfg['run_number'] = cfg['run_number']
import_cfg['dataset_number'] = cfg['dataset_number']
import_cfg['folder_num_pre_offset'] = cfg['run_number']//1000
import_cfg['folder_num'] = (
import_cfg['folder_offset'] + cfg['run_number']//1000
)
import_cfg['folder_pattern'] = import_cfg['folder_pattern'].format(
**import_cfg)
import_cfg['run_folder'] = import_cfg['folder_pattern'].format(
**import_cfg)
if isinstance(glob_files, str):
# single string provided
files = glob.glob(glob_files.format(**import_cfg))
else:
# list of file globs provided
files = []
for file_pattern in glob_files:
files.extend(glob.glob(file_pattern.format(**import_cfg)))
# sort files
files = sorted(files)
# ------------------------------
click.echo('Run: {}'.format(run_number))
click.echo('Outfile: {}'.format(outfile))
click.echo('Keys to import: {}'.format(import_cfg['keys_to_import']))
click.echo('input Files:')
for file in files:
click.echo('\t{}'.format(file))
# crate random services
if 'random_service_use_gslrng' not in cfg:
cfg['random_service_use_gslrng'] = False
random_services, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=3,
use_gslrng=cfg['random_service_use_gslrng'])
# --------------------------------------
# Build IceTray
# --------------------------------------
tray = I3Tray()
# import events from another I3-file
if 'num_events' not in import_cfg:
import_cfg['num_events'] = None
tray.AddModule(
ImportEvents,
'ImportEvents',
files=files,
num_events=import_cfg['num_events'],
keys_to_import=import_cfg['keys_to_import'],
rename_dict=import_cfg['rename_dict'],
mctree_name=import_cfg['mctree_name'],
)
# inject coincident muon from some direction as neutrino
if 'mctree_name' not in cfg['veto_muon_injection_config']:
cfg['veto_muon_injection_config']['mctree_name'] = (
import_cfg['mctree_name']
)
tray.AddModule(
InjectSingleVetoMuon, 'InjectSingleVetoMuon',
random_service=random_services[0],
**cfg['veto_muon_injection_config']
)
t_name = import_cfg['mctree_name']
# # rename I3MCTrees so that we can run PROPOSAL
# def rename_keys(frame, rename_dict):
# for key, new_name in rename_dict.items():
# if key in frame:
# frame[new_name] = frame[key]
# del frame[key]
# print('key', key)
# rename_dict = {
# t_name: '_' + t_name,
# t_name + '_preMuonProp': '_' + t_name + '_preMuonProp',
# t_name + '_preMuonProp_RNGState': (
# '_' + t_name + '_preMuonProp_RNGState',
# ),
# t_name + '_veto_muon': 'I3MCTree_preMuonProp',
# }
# tray.AddModule(rename_keys, 'TempRename', rename_dict=rename_dict)
# temporarily save MMCTrackList
tray.AddModule('Rename', keys=['MMCTrackList', '_MMCTrackList'])
# propagate injected muon
cfg['muon_propagation_config'].update({
'InputMCTreeName': t_name + 'VetoMuon_preMuonProp',
'OutputMCTreeName': t_name + 'VetoMuon',
})
tray.AddSegment(segments.PropagateMuons,
'propagate_muons',
RandomService=random_services[1],
**cfg['muon_propagation_config'])
# undo renaming
tray.AddModule('Rename', keys=['MMCTrackList', 'MMCTrackListVetoMuon'])
tray.AddModule('Rename', keys=['_MMCTrackList', 'MMCTrackList'])
# # now revert renaming
# rename_dict = {
# 'I3MCTree': t_name + '_veto_muon',
# 'I3MCTree_preMuonProp': t_name + '_preMuonProp_veto_muon',
# '_' + t_name: t_name,
# '_' + t_name + '_preMuonProp': t_name + '_preMuonProp',
# '_' + t_name + '_preMuonProp_RNGState': (
# t_name + '_preMuonProp_RNGState',
# ),
# }
# tray.AddModule(rename_keys, 'UndoRenaming', rename_dict=rename_dict)
# create combined I3MCTree for CLSIM
tray.AddModule(
CombineMCTrees, 'CombineMCTrees',
tree1=t_name,
tree2=t_name + 'VetoMuon',
output_key='CombinedMuonVetoI3MCTree',
)
# Bias simulation if desired
if 'ApplyBaseSimulationBias' in cfg and cfg['ApplyBaseSimulationBias']:
tray.AddModule(
BaseSimulationBias,
'BaseSimulationBias',
random_service=random_services[2],
**cfg['BaseSimulationBiasSettings']
)
# --------------------------------------
# Distance Splits
# --------------------------------------
if cfg['distance_splits'] is not None:
click.echo('SplittingDistance: {}'.format(
cfg['distance_splits']))
distance_splits = np.atleast_1d(cfg['distance_splits'])
dom_limits = np.atleast_1d(cfg['threshold_doms'])
if len(dom_limits) == 1:
dom_limits = np.ones_like(distance_splits) * cfg['threshold_doms']
oversize_factors = np.atleast_1d(cfg['oversize_factors'])
order = np.argsort(distance_splits)
distance_splits = distance_splits[order]
dom_limits = dom_limits[order]
oversize_factors = oversize_factors[order]
stream_objects = generate_stream_object(distance_splits,
dom_limits,
oversize_factors)
tray.AddModule(OversizeSplitterNSplits,
"OversizeSplitterNSplits",
thresholds=distance_splits,
thresholds_doms=dom_limits,
oversize_factors=oversize_factors)
for stream_i in stream_objects:
outfile_i = stream_i.transform_filepath(outfile)
tray.AddModule("I3Writer",
"writer_{}".format(stream_i.stream_name),
Filename=outfile_i,
Streams=[icetray.I3Frame.DAQ,
icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')],
If=stream_i)
click.echo('Output ({}): {}'.format(stream_i.stream_name,
outfile_i))
else:
click.echo('Output: {}'.format(outfile))
tray.AddModule("I3Writer", "writer",
Filename=outfile,
Streams=[icetray.I3Frame.DAQ,
icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')])
# --------------------------------------
click.echo('Scratch: {}'.format(scratch))
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
cfg = yaml.load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace(' ', '0')
click.echo('Run: {}'.format(run_number))
click.echo('Outfile: {}'.format(outfile))
click.echo('Azimuth: [{},{}]'.format(cfg['azimuth_min'],
cfg['azimuth_max']))
click.echo('Zenith: [{},{}]'.format(cfg['zenith_min'],
cfg['zenith_max']))
click.echo('Energy: [{},{}]'.format(cfg['e_min'],
cfg['e_max']))
# crate random services
random_services, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=2)
# create convex hull
if 'use_convex_hull' in cfg and cfg['use_convex_hull']:
# hardcode icecube corner points
# ToDo: read from geometry file
points = [
[-570.90002441, -125.13999939, 501], # string 31
[-256.14001465, -521.08001709, 501], # string 1
[ 361. , -422.82998657, 501], # string 6
[ 576.36999512, 170.91999817, 501], # string 50
[ 338.44000244, 463.72000122, 501], # string 74
[ 101.04000092, 412.79000854, 501], # string 72
[ 22.11000061, 509.5 , 501], # string 78
[-347.88000488, 451.51998901, 501], # string 75
[-570.90002441, -125.13999939, -502], # string 31
[-256.14001465, -521.08001709, -502], # string 1
[ 361. , -422.82998657, -502], # string 6
[ 576.36999512, 170.91999817, -502], # string 50
[ 338.44000244, 463.72000122, -502], # string 74
[ 101.04000092, 412.79000854, -502], # string 72
[ 22.11000061, 509.5 , -502], # string 78
[-347.88000488, 451.51998901, -502], # string 75
]
convex_hull = ConvexHull(points)
else:
convex_hull = None
if not 'extend_past_hull' in cfg:
cfg['extend_past_hull'] = 0.0
# create muon
muon = create_muon(
azimuth_range=[cfg['azimuth_min'],cfg['azimuth_max']],
zenith_range=[cfg['zenith_min'],cfg['zenith_max']],
energy_range=[cfg['e_min'],cfg['e_max']],
anchor_time_range=cfg['anchor_time_range'],
anchor_x_range=cfg['anchor_x_range'],
anchor_y_range=cfg['anchor_y_range'],
anchor_z_range=cfg['anchor_z_range'],
length_to_go_back=cfg['length_to_go_back'],
convex_hull=convex_hull,
extend_past_hull=cfg['extend_past_hull'],
random_service=random_services[0],
)
#--------------------------------------
# Build IceTray
#--------------------------------------
tray = I3Tray()
tray.AddModule('I3InfiniteSource', 'source',
# Prefix=gcdfile,
Stream=icetray.I3Frame.DAQ)
tray.AddModule(ParticleMultiplier,
'make_particles',
num_events=cfg['n_events_per_run'],
primary= muon)
tray.AddSegment(segments.PropagateMuons,
'propagate_muons',
RandomService=random_services[1],
**cfg['muon_propagation_config'])
#--------------------------------------
# Distance Splits
#--------------------------------------
if cfg['distance_splits'] is not None:
click.echo('SplittingDistance: {}'.format(
cfg['distance_splits']))
distance_splits = np.atleast_1d(cfg['distance_splits'])
dom_limits = np.atleast_1d(cfg['threshold_doms'])
if len(dom_limits) == 1:
dom_limits = np.ones_like(distance_splits) * cfg['threshold_doms']
oversize_factors = np.atleast_1d(cfg['oversize_factors'])
order = np.argsort(distance_splits)
distance_splits = distance_splits[order]
dom_limits = dom_limits[order]
oversize_factors = oversize_factors[order]
stream_objects = generate_stream_object(distance_splits,
dom_limits,
oversize_factors)
tray.AddModule(OversizeSplitterNSplits,
"OversizeSplitterNSplits",
thresholds=distance_splits,
thresholds_doms=dom_limits,
oversize_factors=oversize_factors)
for stream_i in stream_objects:
outfile_i = stream_i.transform_filepath(outfile)
tray.AddModule("I3Writer",
"writer_{}".format(stream_i.stream_name),
Filename=outfile_i,
Streams=[icetray.I3Frame.DAQ,
icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')],
If=stream_i)
click.echo('Output ({}): {}'.format(stream_i.stream_name,
outfile_i))
else:
click.echo('Output: {}'.format(outfile))
tray.AddModule("I3Writer", "writer",
Filename=outfile,
Streams=[icetray.I3Frame.DAQ,
icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')])
#--------------------------------------
click.echo('Scratch: {}'.format(scratch))
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def process_single_stream(cfg, infile, outfile):
click.echo('Input: {}'.format(infile))
hybrid_mode = (cfg['clsim_hybrid_mode']
and cfg['icemodel'].lower() != 'spicelea')
ignore_muon_light = (cfg['clsim_ignore_muon_light']
and cfg['clsim_hybrid_mode'])
click.echo('UseGPUs: {}'.format(cfg['clsim_usegpus']))
click.echo('IceModel: {}'.format(cfg['icemodel']))
if not cfg['icemodel_location'] is None:
click.echo('IceModelLocation: {}'.format(cfg['icemodel_location']))
click.echo('DomOversize {}'.format(cfg['clsim_dom_oversize']))
click.echo('UnshadowedFraction: {0:.2f}'.format(
cfg['clsim_unshadowed_fraction']))
click.echo('HybridMode: {}'.format(hybrid_mode))
click.echo('IgnoreMuonLight: {}'.format(ignore_muon_light))
click.echo('KeepMCPE: {}'.format(cfg['clsim_keep_mcpe']))
click.echo('Output: {}'.format(outfile))
tray = I3Tray()
tray.context['I3FileStager'] = dataio.get_stagers()
random_services, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=process_single_stream.n_streams)
random_service = random_services[process_single_stream.i_th_stream]
tray.context['I3RandomService'] = random_service
tray.Add('I3Reader', FilenameList=[cfg['gcd'], infile])
if hybrid_mode:
cascade_tables = segments.LoadCascadeTables(
IceModel=cfg['icemodel'], TablePath=cfg['spline_table_dir'])
else:
cascade_tables = None
if cfg['clsim_usegpus']:
use_gpus = True
use_cpus = False
else:
use_gpus = True
use_cpus = False
if 'additional_clsim_params' in cfg:
additional_clsim_params = cfg['additional_clsim_params']
else:
additional_clsim_params = {}
if not cfg['clsim_input_is_sliced']:
MCTreeName = "I3MCTree"
MMCTrackListName = "MMCTrackList"
else:
MCTreeName = "I3MCTree_sliced"
MMCTrackListName = None
#use_gpus=False
#use_cpus=True
tray.AddSegment(segments.PropagatePhotons,
"PropagatePhotons",
GCDFile=cfg['gcd'],
RandomService=random_service,
KeepIndividualMaps=cfg['clsim_keep_mcpe'],
IceModel=cfg['icemodel'],
IceModelLocation=cfg['icemodel_location'],
UnshadowedFraction=cfg['clsim_unshadowed_fraction'],
IgnoreMuons=ignore_muon_light,
HybridMode=hybrid_mode,
UseGPUs=use_gpus,
UseAllCPUCores=use_cpus,
DOMOversizeFactor=cfg['clsim_dom_oversize'],
CascadeService=cascade_tables,
**additional_clsim_params)
# tray.AddSegment(clsim.I3CLSimMakeHits, "makeCLSimHits",
# GCDFile = cfg['gcd'],
# PhotonSeriesName = cfg['photonSeriesName'],
# MCTreeName = MCTreeName,
# MMCTrackListName = MMCTrackListName,
# RandomService = random_service,
# MCPESeriesName = cfg['mcpe_series_map'],
# UnshadowedFraction = cfg['clsim_unshadowed_fraction'],
# UseGPUs = use_gpus,
# UseCPUs = use_cpus,
# IceModelLocation = os.path.expandvars("$I3_BUILD/ice-models/resources/models/spice_lea"),
# )
outfile = outfile.replace(' ', '0')
tray.AddModule("I3Writer",
"writer",
Filename=outfile,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
])
tray.Execute()
del tray
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
if int(yaml.__version__[0]) < 5:
# backwards compatibility for yaml versions before version 5
cfg = yaml.load(stream)
else:
cfg = yaml.full_load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
infile = cfg['infile_pattern'].format(**cfg)
infile = infile.replace(' ', '0')
infile = infile.replace('Level0.{}'.format(cfg['previous_step']),
'Level0.{}'.format(cfg['previous_step'] % 10))
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace('Level0.{}'.format(cfg['step']),
'Level0.{}'.format(cfg['step'] % 10))
outfile = outfile.replace(' ', '0')
outfile = outfile.replace('2012_pass2', 'pass2')
print('Outfile != $FINAL_OUT clean up for crashed scripts not possible!')
tray = I3Tray()
tray.context['I3FileStager'] = dataio.get_stagers()
random_services, run_id = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=1)
random_service = random_services[0]
tray.context['I3RandomService'] = random_service
tray.Add('I3Reader', FilenameList=[cfg['gcd_pass2'], infile])
if run_number < cfg['det_pass2_keep_all_upto']:
cfg['det_keep_mc_hits'] = True
cfg['det_keep_propagated_mc_tree'] = True
cfg['det_keep_mc_pulses'] = True
tray.AddSegment(segments.DetectorSim,
"Detector5Sim",
RandomService='I3RandomService',
RunID=run_id,
GCDFile=cfg['gcd_pass2'],
KeepMCHits=cfg['det_keep_mc_hits'],
KeepPropagatedMCTree=cfg['det_keep_propagated_mc_tree'],
KeepMCPulses=cfg['det_keep_mc_pulses'],
SkipNoiseGenerator=cfg['det_skip_noise_generation'],
LowMem=cfg['det_low_mem'],
InputPESeriesMapName=MCPE_SERIES_MAP,
BeaconLaunches=cfg['det_add_beacon_launches'],
FilterTrigger=cfg['det_filter_trigger'])
tray.AddModule("I3Writer",
"EventWriter",
filename=outfile,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation
])
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
cfg = yaml.load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
infile = cfg['infile_pattern'].format(**cfg)
infile = infile.replace(' ', '0')
infile = infile.replace('Level0.{}'.format(cfg['previous_step']),
'Level0.{}'.format(cfg['previous_step'] % 10))
infile = infile.replace('2012_pass2', 'pass2')
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace('Level0.{}'.format(cfg['step']),
'Level0.{}'.format(cfg['step'] % 10))
outfile = outfile.replace(' ', '0')
outfile = outfile.replace('2012_pass2', 'pass2')
print('Outfile != $FINAL_OUT clean up for crashed scripts not possible!')
tray = I3Tray()
"""The main L1 script"""
tray.AddModule('I3Reader',
'i3 reader',
FilenameList=[cfg['gcd_pass2'], infile])
# run online filters
online_kwargs = {}
if SPLINE_TABLES:
online_kwargs.update({
'SplineRecoAmplitudeTable':
os.path.join(SPLINE_TABLES, 'InfBareMu_mie_abs_z20a10.fits'),
'SplineRecoTimingTable':
os.path.join(SPLINE_TABLES, 'InfBareMu_mie_prob_z20a10.fits'),
'hese_followup_base_GCD_filename':
cfg['gcd_pass2'],
})
if cfg['L1_pass2_run_gfu'] is not None:
online_kwargs['gfu_enabled'] = cfg['L1_pass2_run_gfu']
tray.AddSegment(OnlineFilter,
"OnlineFilter",
decode=False,
simulation=True,
vemcal_enabled=False,
**online_kwargs)
# make random service
_, seed = create_random_services(dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=0)
filter_mask_randoms = phys_services.I3GSLRandomService(seed)
# override MinBias Prescale
filterconfigs = filter_globals.filter_pairs + filter_globals.sdst_pairs
print(cfg['L1_min_bias_prescale'])
if cfg['L1_min_bias_prescale']:
for i, filtertuple in enumerate(filterconfigs):
if filtertuple[0] == filter_globals.FilterMinBias:
del filterconfigs[i]
filterconfigs.append(
(filtertuple[0], cfg['L1_min_bias_prescale']))
break
print(filterconfigs)
# Generate filter Masks for all P frames
tray.AddModule(filter_tools.FilterMaskMaker,
"MakeFilterMasks",
OutputMaskName=filter_globals.filter_mask,
FilterConfigs=filterconfigs,
RandomService=filter_mask_randoms)
# Merge the FilterMasks
tray.AddModule("OrPframeFilterMasks",
"make_q_filtermask",
InputName=filter_globals.filter_mask,
OutputName=filter_globals.qfilter_mask)
# Q+P frame specific keep module needs to go first, as KeepFromSubstram
# will rename things, let's rename post keep.
def is_Q(frame):
return frame.Stop == frame.DAQ
simulation_keeps = [
'BackgroundI3MCTree', 'BackgroundI3MCTreePEcounts',
'BackgroundI3MCPESeriesMap', 'BackgroundI3MCTree_preMuonProp',
'BackgroundMMCTrackList', 'BeaconLaunches', 'CorsikaInteractionHeight',
'CorsikaWeightMap', 'EventProperties', 'GenerationSpec',
'I3LinearizedMCTree', 'I3MCTree', 'I3MCTreePEcounts',
'I3MCTree_preMuonProp', 'I3MCPESeriesMap', 'I3MCPulseSeriesMap',
'I3MCPulseSeriesMapParticleIDMap', 'I3MCWeightDict',
'LeptonInjectorProperties', 'MCHitSeriesMap', 'MCPrimary',
'MCPrimaryInfo', 'MMCTrackList', 'PolyplopiaInfo', 'PolyplopiaPrimary',
'RNGState', 'SignalI3MCPEs', 'SimTrimmer', 'TimeShift', 'WIMP_params'
] + muongun_keys
keep_before_merge = filter_globals.q_frame_keeps + [
'InIceDSTPulses', 'IceTopDSTPulses', 'CalibratedWaveformRange',
'UncleanedInIcePulsesTimeRange', 'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange', 'CalibrationErrata',
'SaturationWindows', 'InIceRawData', 'IceTopRawData'
] + simulation_keeps
tray.AddModule("Keep",
"keep_before_merge",
keys=keep_before_merge,
If=is_Q)
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheckAll",
FilterNameList=filter_globals.filter_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedAnyFilter",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def do_save_just_superdst(frame):
if frame.Has("PassedAnyFilter"):
if not frame["PassedAnyFilter"].value:
return True
else:
return False
else:
print("Failed to find key frame Bool!!")
return False
keep_only_superdsts = filter_globals.keep_nofilterpass + [
'PassedAnyFilter', 'InIceDSTPulses', 'IceTopDSTPulses',
'SplitUncleanedInIcePulses', 'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange', 'RNGState'
] + simulation_keeps
tray.AddModule("Keep",
"KeepOnlySuperDSTs",
keys=keep_only_superdsts,
If=do_save_just_superdst)
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheckSDST",
FilterNameList=filter_globals.sdst_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedKeepSuperDSTOnly",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def dont_save_superdst(frame):
if frame.Has("PassedKeepSuperDSTOnly") and \
frame.Has("PassedAnyFilter"):
if frame["PassedAnyFilter"].value:
return False
elif not frame["PassedKeepSuperDSTOnly"].value:
return True
else:
return False
else:
print("Failed to find key frame Bool!!")
return False
tray.AddModule("Keep",
"KeepOnlyDSTs",
keys=filter_globals.keep_dst_only + [
"PassedAnyFilter", "PassedKeepSuperDSTOnly",
filter_globals.eventheader
] + muongun_keys,
If=dont_save_superdst)
tray.AddModule("KeepFromSubstream",
"null_stream",
StreamName=filter_globals.NullSplitter,
KeepKeys=filter_globals.null_split_keeps)
in_ice_keeps = filter_globals.inice_split_keeps + \
filter_globals.onlinel2filter_keeps
in_ice_keeps = in_ice_keeps + [
'I3EventHeader', 'SplitUncleanedInIcePulses',
'TriggerSplitterLaunchWindow', 'I3TriggerHierarchy',
'GCFilter_GCFilterMJD'
] + muongun_keys
tray.AddModule(
"Keep",
"inice_keeps",
keys=in_ice_keeps,
If=which_split(split_name=filter_globals.InIceSplitter),
)
tray.AddModule("KeepFromSubstream",
"icetop_split_stream",
StreamName=filter_globals.IceTopSplitter,
KeepKeys=filter_globals.icetop_split_keeps)
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheck",
FilterNameList=filter_globals.filters_keeping_allraw,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedConventional",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def I3RawDataCleaner(frame):
if not (('PassedConventional' in frame
and frame['PassedConventional'].value == True) or
('SimTrimmer' in frame and frame['SimTrimmer'].value == True)):
frame.Delete('InIceRawData')
frame.Delete('IceTopRawData')
tray.AddModule(I3RawDataCleaner,
"CleanErrataForConventional",
Streams=[icetray.I3Frame.DAQ])
tray.AddModule("I3Writer",
"EventWriter",
filename=outfile,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation
])
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
if int(yaml.__version__[0]) < 5:
# backwards compatibility for yaml versions before version 5
cfg = yaml.load(stream)
else:
cfg = yaml.full_load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace(' ', '0')
click.echo('NEvents: {}'.format(cfg['n_events_per_run']))
click.echo('EMin: {}'.format(cfg['e_min']))
click.echo('EMax: {}'.format(cfg['e_max']))
click.echo('Gamma: {}'.format(cfg['gamma']))
click.echo('ZenithMin: {}'.format(cfg['zenith_min']))
click.echo('ZenithMax: {}'.format(cfg['zenith_max']))
click.echo('AzimuthMin: {}'.format(cfg['azimuth_min']))
click.echo('AzimuthMax: {}'.format(cfg['azimuth_max']))
if cfg['neutrino_flavor'] is None:
click.echo('NeutrinoTypes: {}'.format(cfg['neutrino_types']))
click.echo('PrimaryTypeRatio: {}'.format(cfg['primary_type_ratio']))
else:
click.echo('NeutrinoFlavor: {}'.format(cfg['neutrino_flavor']))
if 'ApplyBaseSimulationBias' in cfg and cfg['ApplyBaseSimulationBias']:
click.echo('Apply simulation bias: True')
else:
click.echo('Apply simulation bias: True')
tray = I3Tray()
if 'ApplyBaseSimulationBias' in cfg and cfg['ApplyBaseSimulationBias']:
n_services = 3
else:
n_services = 2
random_services, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=n_services,
use_gslrng=cfg['random_service_use_gslrng'],
)
random_service, random_service_prop = random_services[:2]
tray.context['I3RandomService'] = random_service
tray.AddModule("I3InfiniteSource",
"TheSource",
Prefix=cfg['gcd'],
Stream=icetray.I3Frame.DAQ)
tray.AddSegment(segments.GenerateNeutrinos,
"GenerateNeutrinos",
RunID=run_number,
RandomService=random_service,
NumEvents=cfg['n_events_per_run'],
FromEnergy=cfg['e_min'] * icetray.I3Units.GeV,
ToEnergy=cfg['e_max'] * icetray.I3Units.GeV,
ZenithRange=[
cfg['zenith_min'] * icetray.I3Units.deg,
cfg['zenith_max'] * icetray.I3Units.deg
],
AzimuthRange=[
cfg['azimuth_min'] * icetray.I3Units.deg,
cfg['azimuth_max'] * icetray.I3Units.deg
],
**cfg['additional_GenerateNeutrinos_settings'])
# propagate muons if config exists in config
# Note: Snowstorm may perform muon propagation internally
if 'muon_propagation_config' in cfg:
tray.AddSegment(segments.PropagateMuons,
'propagate_muons',
RandomService=random_service_prop,
**cfg['muon_propagation_config'])
else:
# In this case we are not propagating the I3MCTree yet, but
# are letting this be done by snowstorm propagation
# We need to add a key named 'I3MCTree', since snowstorm expects this
# It will propagate the particles for us.
tray.AddModule('Rename', keys=['I3MCTree_preMuonProp', 'I3MCTree'])
# Bias simulation if desired
if 'ApplyBaseSimulationBias' in cfg and cfg['ApplyBaseSimulationBias']:
tray.AddModule(BaseSimulationBias,
'BaseSimulationBias',
random_service=random_services[2],
**cfg['BaseSimulationBiasSettings'])
if cfg['distance_splits'] is not None:
import dom_distance_cut as dom_cut
click.echo('Oversizestreams')
stream_objects = dom_cut.generate_stream_object(
cut_distances=cfg['distance_splits'],
dom_limits=cfg['threshold_doms'],
oversize_factors=cfg['oversize_factors'])
tray.AddModule(dom_cut.OversizeSplitterNSplits,
"OversizeSplitterNSplits",
thresholds=cfg['distance_splits'],
thresholds_doms=cfg['threshold_doms'],
oversize_factors=cfg['oversize_factors'],
simulaton_type=cfg['neutrino_flavor'].lower())
for stream_i in stream_objects:
outfile_i = stream_i.transform_filepath(outfile)
click.echo('\t{}'.format(stream_i))
click.echo('\tOutfile: {}'.format(outfile_i))
tray.AddModule("I3Writer",
"writer_{}".format(stream_i.stream_name),
Filename=outfile_i,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
],
If=stream_i)
else:
click.echo('Output: {}'.format(outfile))
tray.AddModule("I3Writer",
"writer",
Filename=outfile,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
])
click.echo('Scratch: {}'.format(scratch))
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
cfg = yaml.load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace(' ', '0')
click.echo('NEvents: {}'.format(cfg['n_events_per_run']))
click.echo('EMin: {}'.format(cfg['e_min']))
click.echo('EMax: {}'.format(cfg['e_max']))
click.echo('Gamma: {}'.format(cfg['gamma']))
click.echo('ZenithMin: {}'.format(cfg['zenith_min']))
click.echo('ZenithMax: {}'.format(cfg['zenith_max']))
click.echo('AzimuthMin: {}'.format(cfg['azimuth_min']))
click.echo('AzimuthMax: {}'.format(cfg['azimuth_max']))
if cfg['neutrino_flavor'] is None:
click.echo('NeutrinoTypes: {}'.format(cfg['neutrino_types']))
click.echo('PrimaryTypeRatio: {}'.format(cfg['primary_type_ratio']))
else:
click.echo('NeutrinoFlavor: {}'.format(cfg['neutrino_flavor']))
click.echo('CrossSections: {}'.format(cfg['cross_sections']))
if not cfg['cross_sections_path'] is None:
click.echo('CrossSectionsPath: {}'.format(cfg['cross_sections_path']))
tray = I3Tray()
random_services, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=2)
random_service, random_service_prop = random_services
tray.context['I3RandomService'] = random_service
tray.AddModule("I3InfiniteSource",
"TheSource",
Prefix=cfg['gcd'],
Stream=icetray.I3Frame.DAQ)
tray.AddSegment(
segments.GenerateNeutrinos,
"GenerateNeutrinos",
RandomService=random_service,
NumEvents=cfg['n_events_per_run'],
SimMode=cfg['simulation_mode'],
VTXGenMode=cfg['vertex_generation_mode'],
InjectionMode=cfg['injection_mode'],
CylinderParams=cfg['cylinder_params'],
AutoExtendMuonVolume=cfg['auto_extend_muon_volume'],
Flavor=cfg['neutrino_flavor'],
# NuTypes = cfg['neutrino_types'], # Only in newer simprod versions
# PrimaryTypeRatio = cfg['primary_type_ratio'], # Only in newer simprod versions
GammaIndex=cfg['gamma'],
FromEnergy=cfg['e_min'] * icetray.I3Units.GeV,
ToEnergy=cfg['e_max'] * icetray.I3Units.GeV,
ZenithRange=[
cfg['zenith_min'] * icetray.I3Units.deg,
cfg['zenith_max'] * icetray.I3Units.deg
],
AzimuthRange=[
cfg['azimuth_min'] * icetray.I3Units.deg,
cfg['azimuth_max'] * icetray.I3Units.deg
],
# UseDifferentialXsection = cfg['use_diff_cross_section'], # Only in newer simprod versions
CrossSections=cfg['cross_sections'],
CrossSectionsPath=cfg['cross_sections_path'],
# ZenithSamplingMode = cfg['zenith_sampling_mode'], # Only in newer simprod versions
)
tray.AddSegment(segments.PropagateMuons,
"PropagateMuons",
RandomService=random_service_prop,
**cfg['muon_propagation_config'])
if cfg['distance_splits'] is not None:
import dom_distance_cut as dom_cut
click.echo('Oversizestreams')
stream_objects = dom_cut.generate_stream_object(
cut_distances=cfg['distance_splits'],
dom_limits=cfg['threshold_doms'],
oversize_factors=cfg['oversize_factors'])
tray.AddModule(dom_cut.OversizeSplitterNSplits,
"OversizeSplitterNSplits",
thresholds=cfg['distance_splits'],
thresholds_doms=cfg['threshold_doms'],
oversize_factors=cfg['oversize_factors'],
simulaton_type=cfg['neutrino_flavor'].lower())
for stream_i in stream_objects:
outfile_i = stream_i.transform_filepath(outfile)
click.echo('\t{}'.format(stream_i))
click.echo('\tOutfile: {}'.format(outfile_i))
tray.AddModule("I3Writer",
"writer_{}".format(stream_i.stream_name),
Filename=outfile_i,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
],
If=stream_i)
else:
click.echo('Output: {}'.format(outfile))
tray.AddModule("I3Writer",
"writer",
Filename=outfile,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
])
click.echo('Scratch: {}'.format(scratch))
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
cfg = yaml.load(stream, Loader=yaml.Loader)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace(' ', '0')
if cfg['distance_splits'] is not None:
click.echo('SplittingDistances: {}'.format(cfg['distance_splits']))
click.echo('Oversizefactors: {}'.format(cfg['oversize_factors']))
click.echo('NEvents: {}'.format(cfg['n_events_per_run']))
click.echo('EMin: {}'.format(cfg['e_min']))
click.echo('EMax: {}'.format(cfg['e_max']))
click.echo('EBreak: {}'.format(cfg['muongun_e_break']))
click.echo('Gamma: {}'.format(cfg['gamma']))
click.echo('ZenithMin: {}'.format(cfg['zenith_min']))
click.echo('ZenithMax: {}'.format(cfg['zenith_max']))
# create convex hull
if 'use_convex_hull' in cfg and cfg['use_convex_hull']:
# hardcode icecube corner points
# ToDo: read from geometry file
points = [
[-570.90002441, -125.13999939, 501], # string 31
[-256.14001465, -521.08001709, 501], # string 1
[361., -422.82998657, 501], # string 6
[576.36999512, 170.91999817, 501], # string 50
[338.44000244, 463.72000122, 501], # string 74
[101.04000092, 412.79000854, 501], # string 72
[22.11000061, 509.5, 501], # string 78
[-347.88000488, 451.51998901, 501], # string 75
[-570.90002441, -125.13999939, -502], # string 31
[-256.14001465, -521.08001709, -502], # string 1
[361., -422.82998657, -502], # string 6
[576.36999512, 170.91999817, -502], # string 50
[338.44000244, 463.72000122, -502], # string 74
[101.04000092, 412.79000854, -502], # string 72
[22.11000061, 509.5, -502], # string 78
[-347.88000488, 451.51998901, -502], # string 75
]
convex_hull = ConvexHull(points)
else:
convex_hull = None
if 'extend_past_hull' not in cfg:
cfg['extend_past_hull'] = 0.0
random_services, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=2)
random_service, random_service_prop = random_services
# create muon
muon = create_muon(
azimuth_range=[cfg['azimuth_min'], cfg['azimuth_max']],
zenith_range=[cfg['zenith_min'], cfg['zenith_max']],
energy_range=[cfg['e_min'], cfg['e_max']],
anchor_time_range=cfg['anchor_time_range'],
anchor_x_range=cfg['anchor_x_range'],
anchor_y_range=cfg['anchor_y_range'],
anchor_z_range=cfg['anchor_z_range'],
length_to_go_back=cfg['length_to_go_back'],
convex_hull=convex_hull,
extend_past_hull=cfg['extend_past_hull'],
random_service=random_services[0],
)
tray = I3Tray()
tray.context['I3RandomService'] = random_service
tray.AddModule("I3InfiniteSource",
"TheSource",
Prefix=cfg['gcd'],
Stream=icetray.I3Frame.DAQ)
if cfg['MuonGenerator'] == 'MuonGunSinglemuons':
tray.AddSegment(segments.GenerateSingleMuons,
"GenerateCosmicRayMuons",
NumEvents=cfg['n_events_per_run'],
FromEnergy=cfg['e_min'] * icetray.I3Units.GeV,
ToEnergy=cfg['e_max'] * icetray.I3Units.GeV,
BreakEnergy=cfg['muongun_e_break'] *
icetray.I3Units.GeV,
GammaIndex=cfg['gamma'],
ZenithRange=[
cfg['zenith_min'] * icetray.I3Units.deg,
cfg['zenith_max'] * icetray.I3Units.deg
])
elif cfg['MuonGenerator'] == 'MuonGunGeneral':
model = MuonGun.load_model(cfg['muongun_model'])
model.flux.min_multiplicity = cfg['muongun_min_multiplicity']
model.flux.max_multiplicity = cfg['muongun_max_multiplicity']
spectrum = MuonGun.OffsetPowerLaw(cfg['gamma'],
cfg['e_min'] * icetray.I3Units.GeV,
cfg['e_min'] * icetray.I3Units.GeV,
cfg['e_max'] * icetray.I3Units.GeV)
surface = MuonGun.Cylinder(1600, 800,
dataclasses.I3Position(31.25, 19.64, 0))
if cfg['muongun_generator'] == 'energy':
scale = MuonGun.BasicSurfaceScalingFunction()
scale.SetSideScaling(4., 17266, 3.41, 1.74)
scale.SetCapScaling(4., 23710, 3.40, 1.88)
generator = MuonGun.EnergyDependentSurfaceInjector(
surface, model.flux, spectrum, model.radius, scale)
elif cfg['muongun_generator'] == 'static':
generator = MuonGun.StaticSurfaceInjector(surface, model.flux,
spectrum, model.radius)
elif cfg['muongun_generator'] == 'floodlight':
generator = MuonGun.Floodlight(
surface=surface,
energyGenerator=spectrum,
cosMin=cfg['muongun_floodlight_min_cos'],
cosMax=cfg['muongun_floodlight_max_cos'],
)
else:
err_msg = 'MuonGun generator {} is not known.'
err_msg += " Must be 'energy','static' or 'floodlight"
raise ValueError(err_msg.format(cfg['muongun_generator']))
tray.Add(MuonGun.segments.GenerateBundles,
'MuonGenerator',
Generator=generator,
NEvents=cfg['n_events_per_run'],
GCDFile=cfg['gcd'])
tray.Add("Rename", keys=["I3MCTree", "I3MCTree_preMuonProp"])
elif cfg['MuonGenerator'] == 'MuonResimulation':
tray.AddModule(ParticleMultiplier,
'make_particles',
num_events=cfg['n_events_per_run'],
primary=muon)
else:
err_msg = 'MuonGenerator {} is not known.'
err_msg += " Must be 'MuonGunSinglemuons','MuonGunGeneral' or 'MuonResimulation"
raise ValueError(err_msg.format(cfg['MuonGenerator']))
# --------------------------------------
# Propagate Muons
# --------------------------------------
tray.AddSegment(segments.PropagateMuons,
"PropagateMuons",
RandomService=random_service_prop,
**cfg['muon_propagation_config'])
# --------------------------------------
# Distance Splits
# --------------------------------------
if cfg['distance_splits'] is not None:
click.echo('SplittingDistance: {}'.format(cfg['distance_splits']))
distance_splits = np.atleast_1d(cfg['distance_splits'])
dom_limits = np.atleast_1d(cfg['threshold_doms'])
if len(dom_limits) == 1:
dom_limits = np.ones_like(distance_splits) * cfg['threshold_doms']
oversize_factors = np.atleast_1d(cfg['oversize_factors'])
order = np.argsort(distance_splits)
distance_splits = distance_splits[order]
dom_limits = dom_limits[order]
oversize_factors = oversize_factors[order]
stream_objects = generate_stream_object(distance_splits, dom_limits,
oversize_factors)
tray.AddModule(OversizeSplitterNSplits,
"OversizeSplitterNSplits",
thresholds=distance_splits,
thresholds_doms=dom_limits,
oversize_factors=oversize_factors)
for stream_i in stream_objects:
outfile_i = stream_i.transform_filepath(outfile)
tray.AddModule("I3Writer",
"writer_{}".format(stream_i.stream_name),
Filename=outfile_i,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
],
If=stream_i)
click.echo('Output ({}): {}'.format(stream_i.stream_name,
outfile_i))
else:
click.echo('Output: {}'.format(outfile))
tray.AddModule("I3Writer",
"writer",
Filename=outfile,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
])
click.echo('Scratch: {}'.format(scratch))
tray.Execute()
del tray
def main(config_file, run_number, scratch):
with open(config_file, 'r') as stream:
if int(yaml.__version__[0]) < 5:
# backwards compatibility for yaml versions before version 5
cfg = yaml.load(stream)
else:
cfg = yaml.full_load(stream)
if 'dictitems' in cfg.keys():
cfg = cfg['dictitems']
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
infile = cfg['infile_pattern'].format(**cfg)
infile = infile.replace(' ', '0')
infile = infile.replace('Level0.{}'.format(cfg['previous_step']),
'Level0.{}'.format(cfg['previous_step'] % 10))
infile = infile.replace('2012_pass2', '2012')
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace('Level0.{}'.format(cfg['step']),
'Level0.{}'.format(cfg['step'] % 10))
outfile = outfile.replace(' ', '0')
outfile = outfile.replace('2012_pass2', '2012')
print('Outfile != $FINAL_OUT clean up for crashed scripts not possible!')
_, seed = create_random_services(dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=0)
tray = I3Tray()
tray.AddModule('I3Reader',
'reader',
FilenameList=[cfg['gcd_2012'], infile],
SkipKeys=[
'I3DST11',
'I3SuperDST',
'I3VEMCalData',
'PoleMuonLlhFit',
'PoleMuonLlhFitCutsFirstPulseCuts',
'PoleMuonLlhFitFitParams',
'CramerRaoPoleL2IpdfGConvolute_2itParams',
'CramerRaoPoleL2MPEFitParams',
'PoleL2IpdfGConvolute_2it',
'PoleL2IpdfGConvolute_2itFitParams',
'PoleL2MPEFit',
'PoleL2MPEFitCuts',
'PoleL2MPEFitFitParams',
'PoleL2MPEFitMuE',
])
class SkipSFrames(icetray.I3ConditionalModule):
S_stream = icetray.I3Frame.Stream('S')
def __init__(self, context):
icetray.I3ConditionalModule.__init__(self, context)
def Configure(self):
self.Register(self.S_stream, self.SFrame)
def SFrame(self, frame):
pass
# tray.AddModule(SkipSFrames,
# "Skip I Frames")
def check_driving_time(frame):
if 'DrivingTime' not in frame:
frame['DrivingTime'] = dataclasses.I3Time(
frame['I3EventHeader'].start_time)
return True
tray.AddModule(check_driving_time,
'DrivingTimeCheck',
Streams=[icetray.I3Frame.DAQ])
# move that old filterMask out of the way
tray.AddModule("Rename",
"filtermaskmover",
Keys=["FilterMask", "OrigFilterMask"])
if cfg['L1_2012_qify']:
tray.AddModule("QConverter", "qify", WritePFrame=False)
def MissingITCheck(frame):
#print "Fixing IceTop RO"
if "IceTopRawData" not in frame:
itrd = dataclasses.I3DOMLaunchSeriesMap()
frame["IceTopRawData"] = itrd
tray.AddModule(MissingITCheck,
'L1_AddIceTopPulses',
Streams=[icetray.I3Frame.DAQ])
if cfg['L1_2012_retrigger']:
# some cleanup first
tray.AddModule("Delete",
"delete_triggerHierarchy",
Keys=["I3TriggerHierarchy", "TimeShift"])
gcd_file = dataio.I3File(cfg['gcd_2012'])
tray.AddSegment(trigger_sim.TriggerSim, "trig", gcd_file=gcd_file)
tray.AddSegment(jeb_filter_2012.BaseProcessing,
"BaseProc",
pulses=filter_globals.CleanedMuonPulses,
decode=False,
simulation=True,
DomLauncher=(not cfg['L1_2012_dom_simulator']))
tray.AddSegment(jeb_filter_2012.MuonFilter,
"MuonFilter",
pulses=filter_globals.CleanedMuonPulses,
If=which_split(split_name=filter_globals.InIceSplitter))
tray.AddSegment(jeb_filter_2012.CascadeFilter,
"CascadeFilter",
pulses=filter_globals.CleanedMuonPulses,
muon_llhfit_name=filter_globals.muon_llhfit,
If=which_split(split_name=filter_globals.InIceSplitter))
tray.AddSegment(jeb_filter_2012.FSSFilter,
"FSSFilter",
pulses=filter_globals.SplitUncleanedInIcePulses,
If=which_split(split_name=filter_globals.InIceSplitter))
tray.AddSegment(jeb_filter_2012.LowUpFilter,
"LowUpFilter",
If=which_split(split_name=filter_globals.InIceSplitter))
tray.AddSegment(jeb_filter_2012.ShadowFilter,
"ShawdowFilters",
If=which_split(split_name=filter_globals.InIceSplitter))
# use the PID as a seed. Good enough?
tray.AddSegment(jeb_filter_2012.GCFilter,
"GCFilter",
mcseed=seed,
If=which_split(split_name=filter_globals.InIceSplitter))
tray.AddSegment(jeb_filter_2012.VEFFilter,
"VEFFilter",
pulses=filter_globals.CleanedMuonPulses,
If=which_split(split_name=filter_globals.InIceSplitter))
if PHOTONICS_DIR is not None:
photonicstabledirmu = os.path.join(PHOTONICS_DIR, 'SPICE1')
photonicsdriverfilemu = os.path.join('driverfiles', 'mu_photorec.list')
else:
photonicstabledirmu = None
photonicsdriverfilemu = None
tray.AddSegment(jeb_filter_2012.OnlineL2Filter,
"OnlineL2",
pulses=filter_globals.CleanedMuonPulses,
llhfit_name=filter_globals.muon_llhfit,
improved_linefit=True,
paraboloid=False,
PhotonicsTabledirMu=photonicstabledirmu,
PhotonicsDriverfileMu_Spice1=photonicsdriverfilemu,
If=which_split(split_name=filter_globals.InIceSplitter))
tray.AddSegment(jeb_filter_2012.DeepCoreFilter,
"DeepCoreFilter",
pulses=filter_globals.SplitUncleanedInIcePulses,
If=which_split(split_name=filter_globals.InIceSplitter))
tray.AddSegment(jeb_filter_2012.EHEFilter,
"EHEFilter",
If=which_split(split_name=filter_globals.NullSplitter))
tray.AddSegment(jeb_filter_2012.MinBiasFilters,
"MinBias",
If=which_split(split_name=filter_globals.NullSplitter))
tray.AddSegment(jeb_filter_2012.SlopFilters,
"SLOP",
If=which_split(split_name=filter_globals.NullSplitter))
tray.AddSegment(jeb_filter_2012.FixedRateTrigFilter,
"FixedRate",
If=which_split(split_name=filter_globals.NullSplitter))
tray.AddSegment(jeb_filter_2012.CosmicRayFilter,
"CosmicRayFilter",
pulseMask=filter_globals.SplitUncleanedITPulses,
If=which_split(split_name=filter_globals.IceTopSplitter))
tray.AddSegment(jeb_filter_2012.DST,
"DSTFilter",
dstname="I3DST12",
pulses=filter_globals.CleanedMuonPulses,
If=which_split(split_name=filter_globals.InIceSplitter))
# make random service
filter_mask_randoms = phys_services.I3GSLRandomService(seed)
# override MinBias Prescale
filterconfigs = filter_globals.filter_pairs + filter_globals.sdst_pairs
if cfg['L1_min_bias_prescale'] is not None:
for i, filtertuple in enumerate(filterconfigs):
if filtertuple[0] == filter_globals.FilterMinBias:
del filterconfigs[i]
filterconfigs.append(
(filtertuple[0], cfg['L1_min_bias_prescale']))
break
click.echo(filterconfigs)
# Generate filter Masks for all P frames
tray.AddModule(filter_tools.FilterMaskMaker,
"MakeFilterMasks",
OutputMaskName=filter_globals.filter_mask,
FilterConfigs=filterconfigs,
RandomService=filter_mask_randoms)
# Merge the FilterMasks
tray.AddModule("OrPframeFilterMasks",
"make_q_filtermask",
InputName=filter_globals.filter_mask,
OutputName=filter_globals.qfilter_mask)
#Q+P frame specific keep module needs to go first, as KeepFromSubstram
#will rename things, let's rename post keep.
def is_Q(frame):
return frame.Stop == frame.DAQ
tray.AddModule(
"Keep",
"keep_before_merge",
keys=filter_globals.q_frame_keeps + [
'InIceDSTPulses', # keep DST pulse masks
'IceTopDSTPulses',
'CalibratedWaveformRange', # keep calibration info
'UncleanedInIcePulsesTimeRange',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange',
'CalibrationErrata',
'SaturationWindows',
'InIceRawData', # keep raw data for now
'IceTopRawData',
'CorsikaWeightMap', # sim keys
'I3MCWeightDict',
'MCHitSeriesMap',
'MMCTrackList',
'I3MCTree',
'I3LinearizedMCTree',
'MCPrimary',
'MCPrimaryInfo',
'TimeShift', # the time shift amount
'WIMP_params', # Wimp-sim
'SimTrimmer', # for SimTrimmer flag
'I3MCPESeriesMap',
'I3MCPulseSeriesMap',
'I3MCPulseSeriesMapParticleIDMap',
] + muongun_keys,
If=is_Q)
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheckAll",
FilterNameList=filter_globals.filter_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedAnyFilter",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def do_save_just_superdst(frame):
if frame.Has("PassedAnyFilter"):
if not frame["PassedAnyFilter"].value:
return True # <- Event failed to pass any filter.
else:
return False # <- Event passed some filter
else:
print "Failed to find key frame Bool!!"
return False
tray.AddModule(
"Keep",
"KeepOnlySuperDSTs",
keys=filter_globals.keep_nofilterpass + [
'PassedAnyFilter',
'InIceDSTPulses',
'IceTopDSTPulses',
'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange',
'CorsikaWeightMap', # sim keys
'I3MCWeightDict',
'MCHitSeriesMap',
'MMCTrackList',
'I3MCTree',
'I3LinearizedMCTree',
'MCPrimary',
'MCPrimaryInfo',
'TimeShift', # the time shift amount
'WIMP_params', # Wimp-sim
'I3MCPESeriesMap',
'I3MCPulseSeriesMap',
'I3MCPulseSeriesMapParticleIDMap',
] + muongun_keys,
If=do_save_just_superdst)
## Now clean up the events that not even the SuperDST filters passed on.
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheckSDST",
FilterNameList=filter_globals.sdst_streams,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedKeepSuperDSTOnly",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
def dont_save_superdst(frame):
if frame.Has("PassedKeepSuperDSTOnly") and frame.Has(
"PassedAnyFilter"):
if frame["PassedAnyFilter"].value:
return False # <- these passed a regular filter, keeper
elif not frame["PassedKeepSuperDSTOnly"].value:
return True # <- Event failed to pass SDST filter.
else:
return False # <- Event passed some SDST filter
else:
print "Failed to find key frame Bool!!"
return False
# backward compatibility
if 'L1_keep_untriggered' in cfg and cfg['L1_keep_untriggered']:
discard_substream_and_keys = False
else:
discard_substream_and_keys = True
if discard_substream_and_keys:
tray.AddModule("Keep",
"KeepOnlyDSTs",
keys=filter_globals.keep_dst_only + [
"PassedAnyFilter", "PassedKeepSuperDSTOnly",
filter_globals.eventheader
] + muongun_keys,
If=dont_save_superdst)
## Frames should now contain only what is needed. now flatten, write/send to server
## Squish P frames back to single Q frame, one for each split:
tray.AddModule(
"KeepFromSubstream",
"null_stream",
StreamName=filter_globals.NullSplitter,
KeepKeys=filter_globals.null_split_keeps,
)
# Keep the P frames for InIce intact
#tray.AddModule("KeepFromSubstream","inice_split_stream",
# StreamName = filter_globals.InIceSplitter,
# KeepKeys = filter_globals.inice_split_keeps + filter_globals.onlinel2filter_keeps,
# )
#
in_ice_keeps = filter_globals.inice_split_keeps + \
filter_globals.onlinel2filter_keeps
in_ice_keeps = in_ice_keeps + [
'I3EventHeader', 'SplitUncleanedInIcePulses',
'SplitUncleanedInIcePulsesTimeRange',
'SplitUncleanedInIceDSTPulsesTimeRange', 'I3TriggerHierarchy',
'GCFilter_GCFilterMJD'
]
tray.AddModule(
"Keep",
"inice_keeps",
keys=in_ice_keeps + muongun_keys,
If=which_split(split_name=filter_globals.InIceSplitter),
)
tray.AddModule(
"KeepFromSubstream",
"icetop_split_stream",
StreamName=filter_globals.IceTopSplitter,
KeepKeys=filter_globals.icetop_split_keeps,
)
tray.AddModule("I3IcePickModule<FilterMaskFilter>",
"filterMaskCheck",
FilterNameList=filter_globals.filters_keeping_allraw,
FilterResultName=filter_globals.qfilter_mask,
DecisionName="PassedConventional",
DiscardEvents=False,
Streams=[icetray.I3Frame.DAQ])
## Clean out the Raw Data when not passing conventional filter
def I3RawDataCleaner(frame):
if not (('PassedConventional' in frame
and frame['PassedConventional'].value == True) or
('SimTrimmer' in frame and frame['SimTrimmer'].value == True)):
frame.Delete('InIceRawData')
frame.Delete('IceTopRawData')
tray.AddModule(I3RawDataCleaner,
"CleanErrataForConventional",
Streams=[icetray.I3Frame.DAQ])
tray.AddModule("I3Writer",
"EventWriter",
filename=outfile,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.TrayInfo,
icetray.I3Frame.Stream('S')
])
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
cfg = yaml.load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
tray = I3Tray()
random_service, random_service_prop, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'])
tray.context['I3RandomService'] = random_service
model = MuonGun.load_model(cfg['muongun_model'])
model.flux.min_multiplicity = cfg['muongun_min_multiplicity']
model.flux.max_multiplicity = cfg['muongun_max_multiplicity']
spectrum = MuonGun.OffsetPowerLaw( cfg['gamma'],
cfg['e_min']*icetray.I3Units.TeV,
cfg['e_min']*icetray.I3Units.TeV,
cfg['e_max']*icetray.I3Units.TeV)
surface = MuonGun.Cylinder(1600, 800,
dataclasses.I3Position(31.25, 19.64, 0))
if cfg['muongun_generator'] == 'energy':
scale = MuonGun.BasicSurfaceScalingFunction()
scale.SetSideScaling(4., 17266, 3.41, 1.74)
scale.SetCapScaling(4., 23710, 3.40, 1.88)
generator = MuonGun.EnergyDependentSurfaceInjector(surface,
model.flux,
spectrum,
model.radius,
scale)
elif cfg['muongun_generator'] == 'static':
generator = MuonGun.StaticSurfaceInjector(surface,
model.flux,
spectrum,
model.radius)
elif cfg['muongun_generator'] =='floodlight':
generator = MuonGun.Floodlight(surface = surface,
energyGenerator=spectrum,
cosMin=cfg['muongun_floodlight_min_cos'],
cosMax=cfg['muongun_floodlight_max_cos'],
)
else:
err_msg = 'MuonGun generator {} is not known.'
err_msg += " Must be 'energy','static' or 'floodlight"
raise ValueError(err_msg.format(cfg['muongun_generator']))
tray.Add(MuonGun.segments.GenerateBundles, 'MuonGenerator',
Generator=generator,
NEvents=cfg['n_events_per_run'],
GCDFile=cfg['gcd'])
tray.Add("Rename", keys=["I3MCTree", "I3MCTree_preMuonProp"])
tray.AddSegment(
segments.PropagateMuons,
"PropagateMuons",
RandomService=random_service_prop)
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace(' ', '0')
outfile = outfile.replace('.bz2', '')
tray.AddModule("I3Writer", "writer",
Filename=outfile,
Streams=[icetray.I3Frame.DAQ,
icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')])
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
if int(yaml.__version__[0]) < 5:
# backwards compatibility for yaml versions before version 5
cfg = yaml.load(stream)
else:
cfg = yaml.full_load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
tray = I3Tray()
random_services, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=2)
random_service, random_service_prop = random_services
tray.context['I3RandomService'] = random_service
model = MuonGun.load_model(cfg['muongun_model'])
model.flux.min_multiplicity = cfg['muongun_min_multiplicity']
model.flux.max_multiplicity = cfg['muongun_max_multiplicity']
spectrum = MuonGun.OffsetPowerLaw(cfg['gamma'],
cfg['e_min'] * icetray.I3Units.GeV,
cfg['e_min'] * icetray.I3Units.GeV,
cfg['e_max'] * icetray.I3Units.GeV)
surface = MuonGun.Cylinder(1600, 800,
dataclasses.I3Position(31.25, 19.64, 0))
if cfg['muongun_generator'] == 'energy':
scale = MuonGun.BasicSurfaceScalingFunction()
scale.SetSideScaling(4., 17266, 3.41, 1.74)
scale.SetCapScaling(4., 23710, 3.40, 1.88)
generator = MuonGun.EnergyDependentSurfaceInjector(
surface, model.flux, spectrum, model.radius, scale)
elif cfg['muongun_generator'] == 'static':
generator = MuonGun.StaticSurfaceInjector(surface, model.flux,
spectrum, model.radius)
elif cfg['muongun_generator'] == 'floodlight':
generator = MuonGun.Floodlight(
surface=surface,
energyGenerator=spectrum,
cosMin=cfg['muongun_floodlight_min_cos'],
cosMax=cfg['muongun_floodlight_max_cos'],
)
else:
err_msg = 'MuonGun generator {} is not known.'
err_msg += " Must be 'energy','static' or 'floodlight"
raise ValueError(err_msg.format(cfg['muongun_generator']))
tray.Add(MuonGun.segments.GenerateBundles,
'MuonGenerator',
Generator=generator,
NEvents=cfg['n_events_per_run'],
GCDFile=cfg['gcd'])
tray.Add("Rename", keys=["I3MCTree", "I3MCTree_preMuonProp"])
tray.AddSegment(segments.PropagateMuons,
"PropagateMuons",
RandomService=random_service_prop,
**cfg['muon_propagation_config'])
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace(' ', '0')
if cfg['distance_splits'] is not None:
click.echo('SplittingDistance: {}'.format(cfg['distance_splits']))
distance_splits = np.atleast_1d(cfg['distance_splits'])
dom_limits = np.atleast_1d(cfg['threshold_doms'])
if len(dom_limits) == 1:
dom_limits = np.ones_like(distance_splits) * cfg['threshold_doms']
oversize_factors = np.atleast_1d(cfg['oversize_factors'])
order = np.argsort(distance_splits)
distance_splits = distance_splits[order]
dom_limits = dom_limits[order]
oversize_factors = oversize_factors[order]
stream_objects = generate_stream_object(distance_splits, dom_limits,
oversize_factors)
tray.AddModule(OversizeSplitterNSplits,
"OversizeSplitterNSplits",
thresholds=distance_splits,
thresholds_doms=dom_limits,
oversize_factors=oversize_factors)
for stream_i in stream_objects:
outfile_i = stream_i.transform_filepath(outfile)
tray.AddModule("I3Writer",
"writer_{}".format(stream_i.stream_name),
Filename=outfile_i,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
],
If=stream_i)
click.echo('Output ({}): {}'.format(stream_i.stream_name,
outfile_i))
else:
click.echo('Output: {}'.format(outfile))
tray.AddModule("I3Writer",
"writer",
Filename=outfile,
Streams=[
icetray.I3Frame.DAQ, icetray.I3Frame.Physics,
icetray.I3Frame.Stream('S'),
icetray.I3Frame.Stream('M')
])
click.echo('Scratch: {}'.format(scratch))
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
def main(cfg, run_number, scratch):
with open(cfg, 'r') as stream:
cfg = yaml.full_load(stream)
cfg['run_number'] = run_number
cfg['run_folder'] = get_run_folder(run_number)
infile = cfg['infile_pattern'].format(**cfg)
infile = infile.replace(' ', '0')
infile = infile.replace('Level0.{}'.format(cfg['previous_step']),
'Level0.{}'.format(cfg['previous_step'] % 10))
if scratch:
outfile = cfg['scratchfile_pattern'].format(**cfg)
else:
outfile = cfg['outfile_pattern'].format(**cfg)
outfile = outfile.replace('Level0.{}'.format(cfg['step']),
'Level0.{}'.format(cfg['step'] % 10))
outfile = outfile.replace(' ', '0')
outfile = outfile.replace('2012_pass2', 'pass2')
print('Outfile != $FINAL_OUT clean up for crashed scripts not possible!')
tray = I3Tray()
tray.context['I3FileStager'] = dataio.get_stagers()
random_services, run_id = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=1,
use_gslrng=cfg['random_service_use_gslrng'])
random_service = random_services[0]
tray.context['I3RandomService'] = random_service
tray.Add('I3Reader', FilenameList=[cfg['gcd_pass2'], infile])
"""
Perform Detector simulation:
https://code.icecube.wisc.edu/projects/icecube/browser/IceCube/
meta-projects/combo/stable/simprod-scripts/python/segments/
DetectorSim.py
"""
# Combine MCPEs from both detectors
if cfg['det_is_genie_simulation']:
tray.Add("Rename", Keys=[MCPE_SERIES_MAP, 'GenieMCPEs'])
tray.Add("I3CombineMCPE",
InputResponses=["GenieMCPEs", "BackgroundMCPEs"],
OutputResponse=MCPE_SERIES_MAP)
tray.Add("Delete", Keys=['BackgroundMCPEs', 'GenieMCPEs'])
if cfg['det_is_icetop_simulation']:
tray.Add("Rename", Keys=[MCPE_SERIES_MAP, 'InIceMCPEs'])
tray.Add("I3CombineMCPE",
InputResponses=["IceTopMCPEs", "InIceMCPEs"],
OutputResponse=MCPE_SERIES_MAP)
tray.Add("Delete", Keys=['InIceMCPEs', 'IceTopMCPEs'])
# Sample a different efficiency
sample_eff = cfg['det_dom_eff_resmapling_sample_efficiency']
generated_eff = cfg['det_dom_eff_resmapling_generated_efficiency']
if sample_eff > 0.0:
if sample_eff > generated_eff:
msg = 'Cannot upscale from GeneratedEfficiency %s to '
msg += 'SampleEfficiency %s'
icecube.icetray.logging.log_fatal(
msg % (sample_eff, generated_eff))
tray.AddSegment(segments.MultiDomEffSample, "resample",
GeneratedEfficiency=generated_eff,
SampleEfficiencies=[sample_eff],
InputSeriesName=MCPE_SERIES_MAP,
DeleteOriginalSeries=True,
OverwriteOriginalSeries=True)
if run_number < cfg['det_keep_all_upto']:
cfg['det_keep_mc_hits'] = True
cfg['det_keep_propagated_mc_tree'] = True
cfg['det_keep_mc_pulses'] = True
tray.AddSegment(segments.DetectorSim, "DetectorSim",
RandomService='I3RandomService',
RunID=run_id,
GCDFile=cfg['gcd_pass2'],
KeepMCHits=cfg['det_keep_mc_hits'],
KeepPropagatedMCTree=cfg['det_keep_propagated_mc_tree'],
KeepMCPulses=cfg['det_keep_mc_pulses'],
SkipNoiseGenerator=cfg['det_skip_noise_generation'],
LowMem=cfg['det_low_mem'],
InputPESeriesMapName=MCPE_SERIES_MAP,
BeaconLaunches=cfg['det_add_beacon_launches'],
FilterTrigger=cfg['det_filter_trigger'],
TimeShiftSkipKeys=[
"SnowstormParameterRanges",
"SnowstormParameters",
"SnowstormParametrizations",
"SnowstormProposalDistribution",
"WavelengthAcceptance",
"WavelengthGenerationBias",
"LeptonInjectorProperties",
"EventProperties",
"MediumProperties",
],
)
if not cfg['det_keep_mc_pulses']:
tray.Add("Delete", Keys=['I3MCPulseSeriesMapPrimaryIDMap'])
if not cfg['det_keep_mc_hits']:
tray.Add("Delete", Keys=['I3MCPESeriesMapParticleIDMap'])
if cfg['det_remove_keys_from_m_frame']:
tray.Add("Delete", Keys=cfg['det_remove_keys_from_m_frame'])
if cfg['det_convert_to_linear_tree']:
tray.AddModule(segments.ConvertToLinearizedMCTree,
"lineartree", streams=[icetray.I3Frame.DAQ])
tray.AddModule("I3Writer", "EventWriter",
filename=outfile,
Streams=[icetray.I3Frame.DAQ,
icetray.I3Frame.Physics,
icetray.I3Frame.TrayInfo,
icetray.I3Frame.Simulation,
icetray.I3Frame.Stream('m'),
icetray.I3Frame.Stream('M')])
tray.AddModule("TrashCan", "the can")
tray.Execute()
tray.Finish()
