def I3CLSimMakePhotons(tray, name,
GCDFile,
UseCPUs=False,
UseGPUs=True,
UseOnlyDeviceNumber=None,
UseCUDA=False,
MCTreeName="I3MCTree",
OutputMCTreeName=None,
FlasherInfoVectName=None,
FlasherPulseSeriesName=None,
PhotonSeriesName="PhotonSeriesMap",
MCPESeriesName="MCPESeriesMap",
RandomService=None,
IceModelLocation=expandvars("$I3_BUILD/ice-models/resources/models/spice_mie"),
DisableTilt=False,
UnWeightedPhotons=False,
UnWeightedPhotonsScalingFactor=None,
UseI3PropagatorService=True,
UseGeant4=False,
ParticleHistory=False,
ParticleHistoryGranularity=20*icetray.I3Units.m,
CrossoverEnergyEM=None,
CrossoverEnergyHadron=None,
UseCascadeExtension=True,
StopDetectedPhotons=True,
PhotonHistoryEntries=0,
DoNotParallelize=False,
EnableDoubleBuffering=False,
DoublePrecision=False,
DOMOversizeFactor=5.,
UnshadowedFraction=0.9,
HoleIceParameterization=expandvars("$I3_BUILD/ice-models/resources/models/angsens/as.h2-50cm"),
WavelengthAcceptance=None,
DOMRadius=0.16510*icetray.I3Units.m, # 13" diameter
CableOrientation=None,
OverrideApproximateNumberOfWorkItems=None,
IgnoreSubdetectors=['IceTop'],
ExtraArgumentsToI3CLSimClientModule=dict(),
If=lambda f: True
):
"""Do standard clsim processing up to the I3Photon level.
These photons still need to be converted to I3MCPEs to be usable
for further steps in the standard IceCube MC processing chain.
Reads its particles from an I3MCTree and writes an I3PhotonSeriesMap.
All available OpenCL GPUs (and optionally CPUs) will
be used. This will take over your entire machine,
so make sure to configure your batch jobs correctly
when using this on a cluster.
When using nVidia cards, you can set the
CUDA_VISIBLE_DEVICES environment variable to
limit GPU visibility. A setting of
CUDA_VISIBLE_DEVICES="0,3" would only use cards
#0 and #3 and ignore cards #1 and #2. In case you are
using a batch system, chances are this variable is already
set. Unfortunately, there is no corresponding setting
for the AMD driver.
This segment assumes that MMC has been applied to the
I3MCTree and that MMC was *NOT* run using the "-recc" option.
:param UseCPUs:
Turn this on to also use CPU-based devices.
(This may potentially slow down photon generation, which
is also done on the CPU in parallel.)
:param UseGPUs:
Turn this off to not use GPU-based devices.
This may be useful if your GPU is used for display
purposes and you don't want it to slow down.
:param UseOnlyDeviceNumber:
Use only a single device number, even if there is more than
one device found matching the required description. The numbering
starts at 0.
:param UseCUDA:
Use CUDA implementation of photon propagator instead of OpenCL.
:param MCTreeName:
The name of the I3MCTree containing the particles to propagate.
:param OutputMCTreeName:
A copy of the (possibly sliced) MCTree will be stored as this name.
:param FlasherInfoVectName:
Set this to the name of I3FlasherInfoVect objects in the frame to
enable flasher simulation. The module will read I3FlasherInfoVect objects
and generate photons according to assumed parameterizations.
:param FlasherPulseSeriesName:
Set this to the name of an I3CLSimFlasherPulseSeries object in the frame to
enable flasher/Standard Candle simulation.
This cannot be used at the same time as FlasherInfoVectName.
(I3CLSimFlasherPulseSeries objects are clsim's internal flasher
representation, if "FlasherInfoVectName" is used, the I3FlasherInfoVect
objects are converted to I3CLSimFlasherPulseSeries objects.)
:param PhotonSeriesName:
Configure this to enable writing an I3PhotonSeriesMap containing
all photons that reached the DOM surface.
:param RandomService:
Set this to an instance of a I3RandomService. Alternatively,
you can specify the name of a configured I3RandomServiceFactory
added to I3Tray using tray.AddService(). If you don't configure
this, the default I3RandomServiceFactory will be used.
:param IceModelLocation:
Set this either to a directory containing a PPC-compatible
ice description (icemodel.dat, icemodel.par and cfg.txt) or
to a photonics ice table file. PPC-compatible ice files should
generally lead to faster execution times on GPUs since it involves
less interpolation between table entries (the PPC ice-specification
is parametric w.r.t. wavelength, whereas the photonics specification
is not).
:param DisableTilt:
Do not simulate ice tilt, even if the ice model directory
provides tilt information. (Photonics-based models will never
have tilt.)
:param UnWeightedPhotons:
Enabling this setting will disable all optimizations. These
are currently a DOM oversize factor of 5 (with the appropriate
timing correction) and a biased initial photon spectrum that
includes the DOM spectral acceptance. Enabling this setting
essentially means that all photons that would be generated
in the real detector *will* actually be generated. This will siginificatly
slow down the simulation, but the optional ``PhotonSeries``
will contain an unweighted sample of photons that arrive
at your DOMs. This can be useful for DOM acceptance studies.
:param UnWeightedPhotonsScalingFactor:
If UnWeightedPhotons is turned on, this can be used to scale
down the overall number of photons generated. This should normally
not be touched (it can be used when generating photon paths
for animation). Valid range is a float >0. and <=1.
:param StopDetectedPhotons:
Configures behaviour for photons that hit a DOM. If this is true (the default)
photons will be stopped once they hit a DOM. If this is false, they continue to
propagate.
:param PhotonHistoryEntries:
The maximum number of scatterings points to be saved for every photon hitting a DOM.
Only the most recent positions are saved, older positions are overwritten if
the maximum size is reached.
:param UseI3PropagatorService:
Use PROPOSAL and cmc to propagate initial particles (i.e. with NaN
lengths) through the detector.
:param UseGeant4:
Enabling this setting will disable all cascade and muon light yield
parameterizations. All particles will sent to Geant4 for a full
simulation. This does **not** apply to muons that do have a length
assigned. These are assumed to have been generated by MMC and
their light is generated according to the usual parameterization.
:param ParticleHistory:
Store secondary particles produced by particle propagators (e.g.
Geant4, PROPOSAL) in the MCTree.
:param ParticleHistoryGranularity:
When storing history in the MCTree, coalesce secondary EM cascades that
are less than this distance apart.
:param CrossoverEnergyEM:
If set it defines the crossover energy between full Geant4 simulations and
light yield parameterizations for electro magnetic cascades. This only works
when UseGeant4 is set to true. It works in conjunction with CrossoverEnergyHadron.
If one of both is set to a positiv value greater 0 (GeV), the hybrid simulation
is used.
If CrossoverEnergyEM is set to None while CrossoverEnergyHadron is set so
hybrid mode is working, GEANT4 is used for EM cascades.
If CrossoverEnergyEM is set to 0 (GeV) while CrossoverEnergyHadron is set
so hybrid mode is working, leptons and EM cascades will use parameterizations
for the whole energy range.
:param CrossoverEnergyHadron:
If set it defines the crossover energy between full Geant4 simulations and
light yield parameterizations for hadronic cascades. This only works when
UseGeant4 is set to true. It works in conjunction with CrossoverEnergyEM.
If one of both is set to a positiv value greater 0 (GeV), the hybrid simulation
is used.
If CrossoverEnergyHadron is set to None while CrossoverEnergyEM is set so
hybrid mode is working, GEANT4 is used for hadronic cascades.
If CrossoverEnergyHadron is set to 0 (GeV) while CrossoverEnergyHadron is
set so hybrid mode is working, hadronic cascades will use parameterizations
for the whole energy range.
:param UseCascadeExtension:
If set, the cascade light emission parameterizations will include
longitudinal extension. Otherwise, parameterized cascades will be
treated as point-like.
:param DoNotParallelize:
Try only using a single work item in parallel when running the
OpenCL simulation. This might be useful if you want to run jobs
in parallel on a batch system. This will only affect CPUs and
will be a no-op for GPUs.
:param DOMOversizeFactor:
Set the DOM oversize factor. To disable oversizing, set this to 1.
:param UnshadowedFraction:
Fraction of photocathode available to receive light (e.g. unshadowed by the cable)
:param HoleIceParameterization:
Set this to a hole ice parameterization file. The default file contains the
coefficients for nominal angular acceptance correction due to hole ice (ice-models
project is required). Use file $I3_BUILD/ice-models/resources/models/angsens/as.nominal
for no hole ice parameterization.
:param WavelengthAcceptance:
If specified, use this wavelength acceptance to scale the generated
Cherenkov spectrum rather than using the DOM acceptance modified for
oversizing and angular acceptance.
:param DOMRadius:
Allow the DOMRadius to be set externally, for things like mDOMs.
:param CableOrientation:
Path to cable orientation file, e.g. $I3_BUILD/ice-models/resources/models/cable_position/orientation.led7.txt.
If set, blocks photons that would have to pass through the best-fit
position of the cable to reach a DOM. This reduces the isotropic
efficiency by ~10%. Set to None to disable simulation of the cable
shadow.
:param OverrideApproximateNumberOfWorkItems:
Allows to override the auto-detection for the maximum number of parallel work items.
You should only change this if you know what you are doing.
:param If:
Python function to use as conditional execution test for segment modules.
:returns: the dictionary of keyword arguments passed to I3CLSimClientModule
"""
from icecube import icetray, dataclasses, phys_services, clsim
# warn the user in case they might have done something they probably don't want
if UnWeightedPhotons and (DOMOversizeFactor != 1.):
print("********************")
print("Enabling the clsim.I3CLSimMakeHits() \"UnWeightedPhotons=True\" option without setting")
print("\"DOMOversizeFactor=1.\" will still apply a constant weighting factor of DOMOversizeFactor**2.")
print("If this is what you want, you can safely ignore this warning.")
print("********************")
if UnshadowedFraction<=0:
raise RuntimeError("UnshadowedFraction must be a positive number")
clsimParams = setupDetector(
GCDFile=GCDFile,
SimulateFlashers=bool(FlasherInfoVectName or FlasherPulseSeriesName),
IceModelLocation=IceModelLocation,
DisableTilt=DisableTilt,
UnWeightedPhotons=UnWeightedPhotons,
UnWeightedPhotonsScalingFactor=UnWeightedPhotonsScalingFactor,
UseI3PropagatorService=UseI3PropagatorService,
UseGeant4=UseGeant4,
CrossoverEnergyEM=CrossoverEnergyEM,
CrossoverEnergyHadron=CrossoverEnergyHadron,
UseCascadeExtension=UseCascadeExtension,
StopDetectedPhotons=StopDetectedPhotons,
DOMOversizeFactor=DOMOversizeFactor,
UnshadowedFraction=UnshadowedFraction,
HoleIceParameterization=HoleIceParameterization,
WavelengthAcceptance=WavelengthAcceptance,
DOMRadius=DOMRadius,
CableOrientation=CableOrientation,
IgnoreSubdetectors=IgnoreSubdetectors,
)
converters = setupPropagators(RandomService, clsimParams,
UseGPUs=UseGPUs,
UseCPUs=UseCPUs,
OverrideApproximateNumberOfWorkItems=OverrideApproximateNumberOfWorkItems,
DoNotParallelize=DoNotParallelize,
UseOnlyDeviceNumber=UseOnlyDeviceNumber,
EnableDoubleBuffering=EnableDoubleBuffering,
DoublePrecision=DoublePrecision,
UseCUDA=UseCUDA,
)
# bind to a random port on localhost
server = clsim.I3CLSimServer('tcp://127.0.0.1:*', clsim.I3CLSimStepToPhotonConverterSeries(converters))
address = server.GetAddress()
logging.log_info("Server listening at {}".format(address), unit="clsim")
# stash server instance in the context to keep it alive
tray.context[name+'CLSimServer'] = server
if UseGPUs:
if UseI3PropagatorService:
logging.log_warn("Propagating muons and photons in the same process. This may starve your GPU.", unit="clsim")
if UseGeant4:
logging.log_warn("Running Geant and photon propagation in the same process. This will likely starve your GPU.", unit="clsim")
module_config = \
tray.Add(I3CLSimMakePhotonsWithServer, name,
ServerAddress=address,
DetectorSettings=clsimParams,
MCTreeName=MCTreeName,
OutputMCTreeName=OutputMCTreeName,
FlasherInfoVectName=FlasherInfoVectName,
FlasherPulseSeriesName=FlasherPulseSeriesName,
PhotonSeriesName=PhotonSeriesName,
MCPESeriesName=MCPESeriesName,
RandomService=RandomService,
ParticleHistory=ParticleHistory,
ParticleHistoryGranularity=ParticleHistoryGranularity,
ExtraArgumentsToI3CLSimClientModule=ExtraArgumentsToI3CLSimClientModule,
If=If,
)
class GatherStatistics(icetray.I3Module):
"""Mimick the summary stage of I3CLSimModule::Finish()"""
def Finish(self):
if not 'I3SummaryService' in self.context:
return
summary = self.context['I3SummaryService']
server = self.context[name+'CLSimServer']
prefix = 'I3CLSimModule_'+name+'_makePhotons_clsim_'
for k, v in server.GetStatistics().items():
summary[prefix+k] = v
tray.Add(GatherStatistics)
return module_config
tray.Add(populate_s_frame, Streams=[icetray.I3Frame.Stream('S')])
# Add Bumper to stop the tray after NumEventsPerModel Q-frames
tray.Add(Bumper, NumFrames=NumEventsPerModel)
# initialize CLSim server and setup the propagators
converters = setupPropagators(RandomService,
config,
UseGPUs=UseGPUs,
UseCPUs=UseCPUs,
OverrideApproximateNumberOfWorkItems=
OverrideApproximateNumberOfWorkItems,
DoNotParallelize=DoNotParallelize,
UseOnlyDeviceNumber=UseOnlyDeviceNumber)
server = clsim.I3CLSimServer(
"tcp://127.0.0.1:*",
clsim.I3CLSimStepToPhotonConverterSeries(converters))
server_location = server.GetAddress()
# stash server instance in the context to keep it alive
tray.context['CLSimServer'] = server
# recycle StepGenerator to prevent repeated, expensive initialization
if 'StepGenerator' in ExtraArgumentsToI3CLSimClientModule:
stepGenerator = ExtraArgumentsToI3CLSimClientModule['StepGenerator']
stepGenerator.SetMediumProperties(config['MediumProperties'])
stepGenerator.SetWlenBias(config['WavelengthGenerationBias'])
# add CLSim server to tray
module_config = \
tray.Add(I3CLSimMakePhotonsWithServer,
getattr(photon, attr),
err_msg='{} not equal'.format(attr))
dummy_history = dummy_photon_history(photon)
testing.assert_equal(len(dummy_history), len(history))
for pos, expected_pos in zip(history, dummy_history):
testing.assert_equal(pos, expected_pos)
# First, ensure that the test passes when the converter is called directly
test_client(DummyConverter(), 10)
# Now, call through the server in a separate process
converters = clsim.I3CLSimStepToPhotonConverterSeries([DummyConverter()])
address = 'ipc:///tmp/clsim-server.ipc'
server = clsim.I3CLSimServer(address, converters)
def fire_a_few():
client = clsim.I3CLSimClient(address)
testing.assert_equal(client.workgroupSize, 1)
testing.assert_equal(client.maxNumWorkitems, 64)
test_client(client, 10)
procs = [Process(target=fire_a_few) for i in range(3)]
for p in procs:
p.start()
for p in procs:
p.join()
assert p.exitcode == 0
def run_snowstorm_propagation(cfg, infile, outfile):
"""Run SnowStorm Propagation.
Adopted from:
https://code.icecube.wisc.edu/projects/icecube/browser/IceCube/
meta-projects/combo/stable/simprod-scripts/resources/scripts/
SnowSuite/3-Snowstorm.py
Parameters
----------
cfg : dict
Dictionary with configuration settings.
infile : str
Path to input file.
outfile : str
Path to output file.
"""
start_time = time.time()
# --------
# Settings
# --------
default_args = {
# required
'NumEventsPerModel': 100,
'DOMOversizeFactor': 1.,
'UseI3PropagatorService': True,
# optional
'UseGPUs': True,
'SummaryFile': 'summary_snowstorm.yaml',
'UseOnlyDeviceNumber': None,
'MCTreeName': 'I3MCTree',
'OutputMCTreeName': None,
'FlasherInfoVectName': None,
'FlasherPulseSeriesName': None,
'PhotonSeriesName': None,
'MCPESeriesName': "I3MCPESeriesMap",
'DisableTilt': False,
'UnWeightedPhotons': False,
'UnWeightedPhotonsScalingFactor': None,
'UseGeant4': False,
'ParticleHistory': True,
'ParticleHistoryGranularity': 1*icetray.I3Units.m,
'CrossoverEnergyEM': None,
'CrossoverEnergyHadron': None,
'UseCascadeExtension': True,
'StopDetectedPhotons': True,
'PhotonHistoryEntries': 0,
'DoNotParallelize': False,
'UnshadowedFraction': 1.0,
'WavelengthAcceptance': None,
'DOMRadius': 0.16510*icetray.I3Units.m,
'CableOrientation': None,
'OverrideApproximateNumberOfWorkItems': None,
'IgnoreSubdetectors': ["IceTop"],
'ExtraArgumentsToI3CLSimClientModule': dict(),
}
# overwrite default settings
default_args.update(cfg)
cfg = default_args
snowstorm_config = cfg['snowstorm_config']
if cfg['SummaryFile'] is not None:
cfg['SummaryFile'] = cfg['SummaryFile'].format(**cfg)
ice_model_location = \
os.path.expandvars(snowstorm_config["IceModelLocation"])
hole_ice_parameterization = \
os.path.expandvars(snowstorm_config["HoleIceParameterization"])
# set units to meter
cfg['ParticleHistoryGranularity'] *= icetray.I3Units.m
cfg['DOMRadius'] *= icetray.I3Units.m
# Print out most important settings
click.echo('\n---------------')
click.echo('Script Settigns')
click.echo('---------------')
click.echo('\tInput: {}'.format(infile))
click.echo('\tGCDFile: {}'.format(cfg['gcd']))
click.echo('\tOutput: {}'.format(outfile))
for key in ['DOMOversizeFactor', 'UseI3PropagatorService', 'UseGPUs',
'SummaryFile']:
click.echo('\t{}: {}'.format(key, cfg[key]))
click.echo('---------------\n')
# get random service
random_services, _ = create_random_services(
dataset_number=cfg['dataset_number'],
run_number=cfg['run_number'],
seed=cfg['seed'],
n_services=1,
use_gslrng=cfg['random_service_use_gslrng'])
random_service = random_services[0]
"""
Setup and run Snowstorm (aka MultiSim) by running a series of short
trays, each with a different ice model. This works by front-loading as much
of the expensive initialization (reading the GCD file, setting up
PROPOSAL/Geant4, etc) as possible, so that only the propagation kernel
needs to be recompiled for every tray.
"""
# instantiate baseline detector setup.
# this will help construct the baseline characteristics before applying
# the perturbers
print("Setting up detector... ", end="")
clsimParams = setupDetector(
GCDFile=cfg['gcd'],
SimulateFlashers=bool(cfg['FlasherInfoVectName'] or
cfg['FlasherPulseSeriesName']),
IceModelLocation=ice_model_location,
DisableTilt=cfg['DisableTilt'],
UnWeightedPhotons=cfg['UnWeightedPhotons'],
UnWeightedPhotonsScalingFactor=cfg['UnWeightedPhotonsScalingFactor'],
UseI3PropagatorService=cfg['UseI3PropagatorService'],
UseGeant4=cfg['UseGeant4'],
CrossoverEnergyEM=cfg['CrossoverEnergyEM'],
CrossoverEnergyHadron=cfg['CrossoverEnergyHadron'],
UseCascadeExtension=cfg['UseCascadeExtension'],
StopDetectedPhotons=cfg['StopDetectedPhotons'],
DOMOversizeFactor=cfg['DOMOversizeFactor'],
UnshadowedFraction=cfg['UnshadowedFraction'],
HoleIceParameterization=hole_ice_parameterization,
WavelengthAcceptance=cfg['WavelengthAcceptance'],
DOMRadius=cfg['DOMRadius'],
CableOrientation=cfg['CableOrientation'],
IgnoreSubdetectors=cfg['IgnoreSubdetectors'],
)
print("done")
print("Setting up OpenCLDevices... ", end="")
openCLDevices = configureOpenCLDevices(
UseGPUs=cfg['UseGPUs'],
UseCPUs=not cfg['UseGPUs'],
OverrideApproximateNumberOfWorkItems=cfg[
'OverrideApproximateNumberOfWorkItems'],
DoNotParallelize=cfg['DoNotParallelize'],
UseOnlyDeviceNumber=cfg['UseOnlyDeviceNumber'])
print("done")
# -------------------
# Setup perturbations
# -------------------
# create empty "perturber" object
perturber = Perturber()
# get perturbation_cfg dict to simplify calls
perturbation_cfg = snowstorm_config["Perturbations"]
# loop over all perturbations in the perturbation_cfg
print("Setting up perturbers... ")
for name, params in perturbation_cfg.items():
# catch special case of IceWavePlusModes
if name == "IceWavePlusModes":
if not params["apply"]:
continue
if params["type"] == "default":
print("-> adding {} of type {}".format(name, params["type"]))
perturber.add('IceWavePlusModes',
*icewave.get_default_perturbation())
continue
elif hasattr(snowstorm_perturbers, params["type"]):
print("-> adding {} of type {}".format(name, params["type"]))
get_perturber = getattr(snowstorm_perturbers, params["type"])
perturber.add('IceWavePlusModes',
*get_perturber(**params['settings']))
continue
else:
msg = "IceWavePlusModes of type '{}' are not implemented(yet)."
raise NotImplementedError(msg.format(params["type"]))
# all other cases
if params["type"] == "delta":
print("-> adding {} of type {}".format(name, params["type"]))
params = params["delta"]
perturber.add(name, all_parametrizations[name],
DeltaDistribution(params["x0"]))
elif params["type"] == "gauss":
print("-> adding {} of type {}".format(name, params["type"]))
params = params["gauss"]
# Caution: MultivariateNormal expect the covariance matrix as
# first argument, so we need to use sigma**2
perturber.add(name, all_parametrizations[name],
MultivariateNormal(
dataclasses.I3Matrix(np.diag(params["sigma"])**2),
params["mu"]))
elif params["type"] == "uniform":
print("-> adding {} of type {}".format(name, params["type"]))
params = params["uniform"]
perturber.add(name, all_parametrizations[name],
UniformDistribution(
[dataclasses.make_pair(*limits)
for limits in params["limits"]]))
else:
msg = "Perturbation '{}' of type '{}' not implemented."
raise NotImplementedError(msg.format(name, params["type"]))
print("done")
# Setting up some other things
gcdFrames = list(dataio.I3File(cfg['gcd']))
inputStream = dataio.I3FrameSequence([infile])
summary = dataclasses.I3MapStringDouble()
intermediateOutputFiles = []
# --------------
# Run PhotonProp
# --------------
# start a model counter
model_counter = 0
# Execute photon propagation
print("Executing photon propagation...", end="")
while inputStream.more():
# measure CLSimInit time
time_CLSimInit_start = time.time()
tray = I3Tray()
tray.context['I3RandomService'] = random_service
tray.context['I3SummaryService'] = summary
# make a mutable copy of the config dict
config = dict(clsimParams)
# populate the M frame with I3FrameObjects from clsimParams
model = icetray.I3Frame('M')
for k, v in config.items():
if isinstance(v, icetray.I3FrameObject):
model[k] = v
# apply perturbations in the order they were configured
perturber.perturb(random_service, model)
# check for items in the M-frame that were changed/added
# by the perturbers
for k in model.keys():
if k.startswith('Snowstorm'):
# keep all Snowstorm keys
continue
if k not in config:
msg = "\n {} was put in the M frame, but does not appear in "
msg += "the CLSim configuration dict"
raise KeyError(msg.format(k))
if config[k] != model[k]:
# if an items was changed, copy it back to clsimParams
config[k] = model[k]
else:
# remove unmodified items from the M frame
del model[k]
# add "persistent" I3Reader
tray.Add(FrameSequenceReader,
Sequence=itertools.chain(gcdFrames, [model], inputStream))
# inject an S frame if it doesn't exist
tray.Add(EnsureSFrame, Enable=len(intermediateOutputFiles) == 0)
# write pertubations to frame
def populate_s_frame(frame):
perturber.to_frame(frame)
tray.Add(populate_s_frame, Streams=[icetray.I3Frame.Stream('S')])
# Add Bumper to stop the tray after NumEventsPerModel Q-frames
tray.Add(Bumper, NumFrames=cfg['NumEventsPerModel'])
# initialize CLSim server and setup the propagators
server_location = tempfile.mkstemp(prefix='clsim-server-')[1]
address = 'ipc://'+server_location
converters = setupPropagators(
random_service, config,
UseGPUs=cfg['UseGPUs'],
UseCPUs=not cfg['UseGPUs'],
OverrideApproximateNumberOfWorkItems=cfg[
'OverrideApproximateNumberOfWorkItems'],
DoNotParallelize=cfg['DoNotParallelize'],
UseOnlyDeviceNumber=cfg['UseOnlyDeviceNumber']
)
server = clsim.I3CLSimServer(
address, clsim.I3CLSimStepToPhotonConverterSeries(converters))
# stash server instance in the context to keep it alive
tray.context['CLSimServer'] = server
# recycle StepGenerator to prevent repeated, expensive initialization
if 'StepGenerator' in cfg['ExtraArgumentsToI3CLSimClientModule']:
stepGenerator = \
cfg['ExtraArgumentsToI3CLSimClientModule']['StepGenerator']
stepGenerator.SetMediumProperties(config['MediumProperties'])
stepGenerator.SetWlenBias(config['WavelengthGenerationBias'])
# add CLSim server to tray
module_config = \
tray.Add(
I3CLSimMakePhotonsWithServer,
ServerAddress=address,
DetectorSettings=config,
MCTreeName=cfg['MCTreeName'],
OutputMCTreeName=cfg['OutputMCTreeName'],
FlasherInfoVectName=cfg['FlasherInfoVectName'],
FlasherPulseSeriesName=cfg['FlasherPulseSeriesName'],
PhotonSeriesName=cfg['PhotonSeriesName'],
MCPESeriesName=cfg['MCPESeriesName'],
RandomService=random_service,
ParticleHistory=cfg['ParticleHistory'],
ParticleHistoryGranularity=cfg['ParticleHistoryGranularity'],
ExtraArgumentsToI3CLSimClientModule=cfg[
'ExtraArgumentsToI3CLSimClientModule'],
)
# recycle StepGenerator to prevent repeated, expensive initialization
cfg['ExtraArgumentsToI3CLSimClientModule']['StepGenerator'] = \
module_config['StepGenerator']
# write to temporary output file
intermediateOutputFiles.append(
tempfile.mkstemp(suffix=(outfile.split("/"))[-1])[1])
tray.Add("I3Writer",
Filename=intermediateOutputFiles[-1],
DropOrphanStreams=[icetray.I3Frame.TrayInfo],
Streams=[icetray.I3Frame.TrayInfo,
icetray.I3Frame.Simulation,
icetray.I3Frame.Stream('M'),
icetray.I3Frame.Stream('m'),
icetray.I3Frame.DAQ,
icetray.I3Frame.Physics])
# gather statistics in the "I3SummaryService"
tray.Add(GatherStatistics)
# measure CLSimInit time
time_CLSimInit = time.time() - time_CLSimInit_start
summary["CLSimInitTime_{:03d}".format(model_counter)] = time_CLSimInit
if "TotalCLSimInitTime" not in summary:
summary["TotalCLSimInitTime"] = time_CLSimInit
else:
summary["TotalCLSimInitTime"] += time_CLSimInit
# measure CLSimTray time
time_CLSimTray_start = time.time()
# Execute Tray
tray.Execute()
# measure CLSimTray time
time_CLSimTray = time.time() - time_CLSimTray_start
summary["CLSimTrayTime_{:03d}".format(model_counter)] = time_CLSimTray
if "TotalCLSimTrayTime" not in summary:
summary["TotalCLSimTrayTime"] = time_CLSimTray
else:
summary["TotalCLSimTrayTime"] += time_CLSimTray
# remove the temp file made by the server location thingy
os.unlink(server_location)
# increase model counter
model_counter += 1
print("done")
# Add number of models to summary
summary["TotalNumberOfModels"] = model_counter
# Concatenate intermediate files
print("Concatenating temporary files... ", end='')
tray = I3Tray()
tray.Add(dataio.I3Reader, "I3Reader", FilenameList=intermediateOutputFiles)
tray.Add("I3Writer", Filename=outfile,
DropOrphanStreams=[icetray.I3Frame.TrayInfo],
Streams=[icetray.I3Frame.TrayInfo,
icetray.I3Frame.Simulation,
icetray.I3Frame.Stream('M'),
icetray.I3Frame.Stream('m'),
icetray.I3Frame.DAQ,
icetray.I3Frame.Physics])
tray.Execute()
tray.Finish()
print("done")
print("Cleaning up Temporary files... ")
for fname in intermediateOutputFiles:
os.unlink(fname)
print("done")
# Recalculate averages
print("Writing summary file... ", end='')
if cfg['UseGPUs']:
if summary['TotalHostTime'] > 0.0:
summary['DeviceUtilization'] = \
summary['TotalDeviceTime']/summary['TotalHostTime']
if summary['TotalNumPhotonsGenerated'] > 0.0:
summary['AverageDeviceTimePerPhoton'] = \
summary['TotalDeviceTime']/summary['TotalNumPhotonsGenerated']
if summary['TotalNumPhotonsGenerated'] > 0.0:
summary['AverageHostTimePerPhoton'] = \
summary['TotalHostTime']/summary['TotalNumPhotonsGenerated']
if cfg['SummaryFile']:
with open(cfg['SummaryFile'], 'w') as f:
yaml.dump(dict(summary), f)
print("done")
print('--------')
print('Summary:')
print('--------')
for key, value in summary.items():
print('\t{}: {}'.format(key, value))
print('--------\n')
# Hurray!
print("All finished!")
# say something about the runtime
end_time = time.time()
print("That took "+str(end_time - start_time)+" seconds.")
feed()
t.join()
icetray.logging.log_info("base {} done".format(base), unit='clsim.testCLSimServer')
# icetray.logging.set_level_for_unit('I3CLSimServer', 'TRACE')
# icetray.logging.set_level_for_unit('I3CLSimServer', 'DEBUG')
icetray.logging.set_level_for_unit('I3CLSimClient', 'TRACE')
icetray.logging.set_level_for_unit('clsim.testCLSimServer', 'TRACE')
# First, ensure that the test passes when the converter is called directly
test_client(DummyConverter(), 10)
# Now, call through the server in a separate process
converters = clsim.I3CLSimStepToPhotonConverterSeries([DummyConverter()])
server = clsim.I3CLSimServer('tcp://127.0.0.1:*',converters)
address = server.GetAddress()
def fire_a_few(num_steps=10, base=0):
# NB: the Python logging bridge deadlocks from secondary threads in Py3
if sys.version_info.major == 2:
icetray.logging.BASIC_FORMAT = "{} %(filename)s:%(lineno)s %(levelname)s: %(message)s".format(base)
icetray.logging.console()
icetray.logging.set_level_for_unit('clsim.testCLSimServer', 'TRACE')
icetray.logging.set_level_for_unit('I3CLSimClient', 'TRACE')
icetray.logging.log_debug("client {} connecting to {}".format(base, address), unit='clsim.testCLSimServer')
client = clsim.I3CLSimClient(address)
icetray.logging.log_debug("client {} connected".format(base), unit='clsim.testCLSimServer')
testing.assert_equal( client.workgroupSize, 8 )
testing.assert_equal( client.maxNumWorkitems, 64 )