def I3CLSimMakePhotons(tray, name, GCDFile, UseCPUs=False, UseGPUs=True, UseOnlyDeviceNumber=None, UseCUDA=False, MCTreeName="I3MCTree", OutputMCTreeName=None, FlasherInfoVectName=None, FlasherPulseSeriesName=None, PhotonSeriesName="PhotonSeriesMap", MCPESeriesName="MCPESeriesMap", RandomService=None, IceModelLocation=expandvars("$I3_BUILD/ice-models/resources/models/spice_mie"), DisableTilt=False, UnWeightedPhotons=False, UnWeightedPhotonsScalingFactor=None, UseI3PropagatorService=True, UseGeant4=False, ParticleHistory=False, ParticleHistoryGranularity=20*icetray.I3Units.m, CrossoverEnergyEM=None, CrossoverEnergyHadron=None, UseCascadeExtension=True, StopDetectedPhotons=True, PhotonHistoryEntries=0, DoNotParallelize=False, EnableDoubleBuffering=False, DoublePrecision=False, DOMOversizeFactor=5., UnshadowedFraction=0.9, HoleIceParameterization=expandvars("$I3_BUILD/ice-models/resources/models/angsens/as.h2-50cm"), WavelengthAcceptance=None, DOMRadius=0.16510*icetray.I3Units.m, # 13" diameter CableOrientation=None, OverrideApproximateNumberOfWorkItems=None, IgnoreSubdetectors=['IceTop'], ExtraArgumentsToI3CLSimClientModule=dict(), If=lambda f: True ): """Do standard clsim processing up to the I3Photon level. These photons still need to be converted to I3MCPEs to be usable for further steps in the standard IceCube MC processing chain. Reads its particles from an I3MCTree and writes an I3PhotonSeriesMap. All available OpenCL GPUs (and optionally CPUs) will be used. This will take over your entire machine, so make sure to configure your batch jobs correctly when using this on a cluster. When using nVidia cards, you can set the CUDA_VISIBLE_DEVICES environment variable to limit GPU visibility. A setting of CUDA_VISIBLE_DEVICES="0,3" would only use cards #0 and #3 and ignore cards #1 and #2. In case you are using a batch system, chances are this variable is already set. Unfortunately, there is no corresponding setting for the AMD driver. This segment assumes that MMC has been applied to the I3MCTree and that MMC was *NOT* run using the "-recc" option. :param UseCPUs: Turn this on to also use CPU-based devices. (This may potentially slow down photon generation, which is also done on the CPU in parallel.) :param UseGPUs: Turn this off to not use GPU-based devices. This may be useful if your GPU is used for display purposes and you don't want it to slow down. :param UseOnlyDeviceNumber: Use only a single device number, even if there is more than one device found matching the required description. The numbering starts at 0. :param UseCUDA: Use CUDA implementation of photon propagator instead of OpenCL. :param MCTreeName: The name of the I3MCTree containing the particles to propagate. :param OutputMCTreeName: A copy of the (possibly sliced) MCTree will be stored as this name. :param FlasherInfoVectName: Set this to the name of I3FlasherInfoVect objects in the frame to enable flasher simulation. The module will read I3FlasherInfoVect objects and generate photons according to assumed parameterizations. :param FlasherPulseSeriesName: Set this to the name of an I3CLSimFlasherPulseSeries object in the frame to enable flasher/Standard Candle simulation. This cannot be used at the same time as FlasherInfoVectName. (I3CLSimFlasherPulseSeries objects are clsim's internal flasher representation, if "FlasherInfoVectName" is used, the I3FlasherInfoVect objects are converted to I3CLSimFlasherPulseSeries objects.) :param PhotonSeriesName: Configure this to enable writing an I3PhotonSeriesMap containing all photons that reached the DOM surface. :param RandomService: Set this to an instance of a I3RandomService. Alternatively, you can specify the name of a configured I3RandomServiceFactory added to I3Tray using tray.AddService(). If you don't configure this, the default I3RandomServiceFactory will be used. :param IceModelLocation: Set this either to a directory containing a PPC-compatible ice description (icemodel.dat, icemodel.par and cfg.txt) or to a photonics ice table file. PPC-compatible ice files should generally lead to faster execution times on GPUs since it involves less interpolation between table entries (the PPC ice-specification is parametric w.r.t. wavelength, whereas the photonics specification is not). :param DisableTilt: Do not simulate ice tilt, even if the ice model directory provides tilt information. (Photonics-based models will never have tilt.) :param UnWeightedPhotons: Enabling this setting will disable all optimizations. These are currently a DOM oversize factor of 5 (with the appropriate timing correction) and a biased initial photon spectrum that includes the DOM spectral acceptance. Enabling this setting essentially means that all photons that would be generated in the real detector *will* actually be generated. This will siginificatly slow down the simulation, but the optional ``PhotonSeries`` will contain an unweighted sample of photons that arrive at your DOMs. This can be useful for DOM acceptance studies. :param UnWeightedPhotonsScalingFactor: If UnWeightedPhotons is turned on, this can be used to scale down the overall number of photons generated. This should normally not be touched (it can be used when generating photon paths for animation). Valid range is a float >0. and <=1. :param StopDetectedPhotons: Configures behaviour for photons that hit a DOM. If this is true (the default) photons will be stopped once they hit a DOM. If this is false, they continue to propagate. :param PhotonHistoryEntries: The maximum number of scatterings points to be saved for every photon hitting a DOM. Only the most recent positions are saved, older positions are overwritten if the maximum size is reached. :param UseI3PropagatorService: Use PROPOSAL and cmc to propagate initial particles (i.e. with NaN lengths) through the detector. :param UseGeant4: Enabling this setting will disable all cascade and muon light yield parameterizations. All particles will sent to Geant4 for a full simulation. This does **not** apply to muons that do have a length assigned. These are assumed to have been generated by MMC and their light is generated according to the usual parameterization. :param ParticleHistory: Store secondary particles produced by particle propagators (e.g. Geant4, PROPOSAL) in the MCTree. :param ParticleHistoryGranularity: When storing history in the MCTree, coalesce secondary EM cascades that are less than this distance apart. :param CrossoverEnergyEM: If set it defines the crossover energy between full Geant4 simulations and light yield parameterizations for electro magnetic cascades. This only works when UseGeant4 is set to true. It works in conjunction with CrossoverEnergyHadron. If one of both is set to a positiv value greater 0 (GeV), the hybrid simulation is used. If CrossoverEnergyEM is set to None while CrossoverEnergyHadron is set so hybrid mode is working, GEANT4 is used for EM cascades. If CrossoverEnergyEM is set to 0 (GeV) while CrossoverEnergyHadron is set so hybrid mode is working, leptons and EM cascades will use parameterizations for the whole energy range. :param CrossoverEnergyHadron: If set it defines the crossover energy between full Geant4 simulations and light yield parameterizations for hadronic cascades. This only works when UseGeant4 is set to true. It works in conjunction with CrossoverEnergyEM. If one of both is set to a positiv value greater 0 (GeV), the hybrid simulation is used. If CrossoverEnergyHadron is set to None while CrossoverEnergyEM is set so hybrid mode is working, GEANT4 is used for hadronic cascades. If CrossoverEnergyHadron is set to 0 (GeV) while CrossoverEnergyHadron is set so hybrid mode is working, hadronic cascades will use parameterizations for the whole energy range. :param UseCascadeExtension: If set, the cascade light emission parameterizations will include longitudinal extension. Otherwise, parameterized cascades will be treated as point-like. :param DoNotParallelize: Try only using a single work item in parallel when running the OpenCL simulation. This might be useful if you want to run jobs in parallel on a batch system. This will only affect CPUs and will be a no-op for GPUs. :param DOMOversizeFactor: Set the DOM oversize factor. To disable oversizing, set this to 1. :param UnshadowedFraction: Fraction of photocathode available to receive light (e.g. unshadowed by the cable) :param HoleIceParameterization: Set this to a hole ice parameterization file. The default file contains the coefficients for nominal angular acceptance correction due to hole ice (ice-models project is required). Use file $I3_BUILD/ice-models/resources/models/angsens/as.nominal for no hole ice parameterization. :param WavelengthAcceptance: If specified, use this wavelength acceptance to scale the generated Cherenkov spectrum rather than using the DOM acceptance modified for oversizing and angular acceptance. :param DOMRadius: Allow the DOMRadius to be set externally, for things like mDOMs. :param CableOrientation: Path to cable orientation file, e.g. $I3_BUILD/ice-models/resources/models/cable_position/orientation.led7.txt. If set, blocks photons that would have to pass through the best-fit position of the cable to reach a DOM. This reduces the isotropic efficiency by ~10%. Set to None to disable simulation of the cable shadow. :param OverrideApproximateNumberOfWorkItems: Allows to override the auto-detection for the maximum number of parallel work items. You should only change this if you know what you are doing. :param If: Python function to use as conditional execution test for segment modules. :returns: the dictionary of keyword arguments passed to I3CLSimClientModule """ from icecube import icetray, dataclasses, phys_services, clsim # warn the user in case they might have done something they probably don't want if UnWeightedPhotons and (DOMOversizeFactor != 1.): print("********************") print("Enabling the clsim.I3CLSimMakeHits() \"UnWeightedPhotons=True\" option without setting") print("\"DOMOversizeFactor=1.\" will still apply a constant weighting factor of DOMOversizeFactor**2.") print("If this is what you want, you can safely ignore this warning.") print("********************") if UnshadowedFraction<=0: raise RuntimeError("UnshadowedFraction must be a positive number") clsimParams = setupDetector( GCDFile=GCDFile, SimulateFlashers=bool(FlasherInfoVectName or FlasherPulseSeriesName), IceModelLocation=IceModelLocation, DisableTilt=DisableTilt, UnWeightedPhotons=UnWeightedPhotons, UnWeightedPhotonsScalingFactor=UnWeightedPhotonsScalingFactor, UseI3PropagatorService=UseI3PropagatorService, UseGeant4=UseGeant4, CrossoverEnergyEM=CrossoverEnergyEM, CrossoverEnergyHadron=CrossoverEnergyHadron, UseCascadeExtension=UseCascadeExtension, StopDetectedPhotons=StopDetectedPhotons, DOMOversizeFactor=DOMOversizeFactor, UnshadowedFraction=UnshadowedFraction, HoleIceParameterization=HoleIceParameterization, WavelengthAcceptance=WavelengthAcceptance, DOMRadius=DOMRadius, CableOrientation=CableOrientation, IgnoreSubdetectors=IgnoreSubdetectors, ) converters = setupPropagators(RandomService, clsimParams, UseGPUs=UseGPUs, UseCPUs=UseCPUs, OverrideApproximateNumberOfWorkItems=OverrideApproximateNumberOfWorkItems, DoNotParallelize=DoNotParallelize, UseOnlyDeviceNumber=UseOnlyDeviceNumber, EnableDoubleBuffering=EnableDoubleBuffering, DoublePrecision=DoublePrecision, UseCUDA=UseCUDA, ) # bind to a random port on localhost server = clsim.I3CLSimServer('tcp://127.0.0.1:*', clsim.I3CLSimStepToPhotonConverterSeries(converters)) address = server.GetAddress() logging.log_info("Server listening at {}".format(address), unit="clsim") # stash server instance in the context to keep it alive tray.context[name+'CLSimServer'] = server if UseGPUs: if UseI3PropagatorService: logging.log_warn("Propagating muons and photons in the same process. This may starve your GPU.", unit="clsim") if UseGeant4: logging.log_warn("Running Geant and photon propagation in the same process. This will likely starve your GPU.", unit="clsim") module_config = \ tray.Add(I3CLSimMakePhotonsWithServer, name, ServerAddress=address, DetectorSettings=clsimParams, MCTreeName=MCTreeName, OutputMCTreeName=OutputMCTreeName, FlasherInfoVectName=FlasherInfoVectName, FlasherPulseSeriesName=FlasherPulseSeriesName, PhotonSeriesName=PhotonSeriesName, MCPESeriesName=MCPESeriesName, RandomService=RandomService, ParticleHistory=ParticleHistory, ParticleHistoryGranularity=ParticleHistoryGranularity, ExtraArgumentsToI3CLSimClientModule=ExtraArgumentsToI3CLSimClientModule, If=If, ) class GatherStatistics(icetray.I3Module): """Mimick the summary stage of I3CLSimModule::Finish()""" def Finish(self): if not 'I3SummaryService' in self.context: return summary = self.context['I3SummaryService'] server = self.context[name+'CLSimServer'] prefix = 'I3CLSimModule_'+name+'_makePhotons_clsim_' for k, v in server.GetStatistics().items(): summary[prefix+k] = v tray.Add(GatherStatistics) return module_config
tray.Add(populate_s_frame, Streams=[icetray.I3Frame.Stream('S')]) # Add Bumper to stop the tray after NumEventsPerModel Q-frames tray.Add(Bumper, NumFrames=NumEventsPerModel) # initialize CLSim server and setup the propagators converters = setupPropagators(RandomService, config, UseGPUs=UseGPUs, UseCPUs=UseCPUs, OverrideApproximateNumberOfWorkItems= OverrideApproximateNumberOfWorkItems, DoNotParallelize=DoNotParallelize, UseOnlyDeviceNumber=UseOnlyDeviceNumber) server = clsim.I3CLSimServer( "tcp://127.0.0.1:*", clsim.I3CLSimStepToPhotonConverterSeries(converters)) server_location = server.GetAddress() # stash server instance in the context to keep it alive tray.context['CLSimServer'] = server # recycle StepGenerator to prevent repeated, expensive initialization if 'StepGenerator' in ExtraArgumentsToI3CLSimClientModule: stepGenerator = ExtraArgumentsToI3CLSimClientModule['StepGenerator'] stepGenerator.SetMediumProperties(config['MediumProperties']) stepGenerator.SetWlenBias(config['WavelengthGenerationBias']) # add CLSim server to tray module_config = \ tray.Add(I3CLSimMakePhotonsWithServer,
getattr(photon, attr), err_msg='{} not equal'.format(attr)) dummy_history = dummy_photon_history(photon) testing.assert_equal(len(dummy_history), len(history)) for pos, expected_pos in zip(history, dummy_history): testing.assert_equal(pos, expected_pos) # First, ensure that the test passes when the converter is called directly test_client(DummyConverter(), 10) # Now, call through the server in a separate process converters = clsim.I3CLSimStepToPhotonConverterSeries([DummyConverter()]) address = 'ipc:///tmp/clsim-server.ipc' server = clsim.I3CLSimServer(address, converters) def fire_a_few(): client = clsim.I3CLSimClient(address) testing.assert_equal(client.workgroupSize, 1) testing.assert_equal(client.maxNumWorkitems, 64) test_client(client, 10) procs = [Process(target=fire_a_few) for i in range(3)] for p in procs: p.start() for p in procs: p.join() assert p.exitcode == 0
def run_snowstorm_propagation(cfg, infile, outfile): """Run SnowStorm Propagation. Adopted from: https://code.icecube.wisc.edu/projects/icecube/browser/IceCube/ meta-projects/combo/stable/simprod-scripts/resources/scripts/ SnowSuite/3-Snowstorm.py Parameters ---------- cfg : dict Dictionary with configuration settings. infile : str Path to input file. outfile : str Path to output file. """ start_time = time.time() # -------- # Settings # -------- default_args = { # required 'NumEventsPerModel': 100, 'DOMOversizeFactor': 1., 'UseI3PropagatorService': True, # optional 'UseGPUs': True, 'SummaryFile': 'summary_snowstorm.yaml', 'UseOnlyDeviceNumber': None, 'MCTreeName': 'I3MCTree', 'OutputMCTreeName': None, 'FlasherInfoVectName': None, 'FlasherPulseSeriesName': None, 'PhotonSeriesName': None, 'MCPESeriesName': "I3MCPESeriesMap", 'DisableTilt': False, 'UnWeightedPhotons': False, 'UnWeightedPhotonsScalingFactor': None, 'UseGeant4': False, 'ParticleHistory': True, 'ParticleHistoryGranularity': 1*icetray.I3Units.m, 'CrossoverEnergyEM': None, 'CrossoverEnergyHadron': None, 'UseCascadeExtension': True, 'StopDetectedPhotons': True, 'PhotonHistoryEntries': 0, 'DoNotParallelize': False, 'UnshadowedFraction': 1.0, 'WavelengthAcceptance': None, 'DOMRadius': 0.16510*icetray.I3Units.m, 'CableOrientation': None, 'OverrideApproximateNumberOfWorkItems': None, 'IgnoreSubdetectors': ["IceTop"], 'ExtraArgumentsToI3CLSimClientModule': dict(), } # overwrite default settings default_args.update(cfg) cfg = default_args snowstorm_config = cfg['snowstorm_config'] if cfg['SummaryFile'] is not None: cfg['SummaryFile'] = cfg['SummaryFile'].format(**cfg) ice_model_location = \ os.path.expandvars(snowstorm_config["IceModelLocation"]) hole_ice_parameterization = \ os.path.expandvars(snowstorm_config["HoleIceParameterization"]) # set units to meter cfg['ParticleHistoryGranularity'] *= icetray.I3Units.m cfg['DOMRadius'] *= icetray.I3Units.m # Print out most important settings click.echo('\n---------------') click.echo('Script Settigns') click.echo('---------------') click.echo('\tInput: {}'.format(infile)) click.echo('\tGCDFile: {}'.format(cfg['gcd'])) click.echo('\tOutput: {}'.format(outfile)) for key in ['DOMOversizeFactor', 'UseI3PropagatorService', 'UseGPUs', 'SummaryFile']: click.echo('\t{}: {}'.format(key, cfg[key])) click.echo('---------------\n') # get random service random_services, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=1, use_gslrng=cfg['random_service_use_gslrng']) random_service = random_services[0] """ Setup and run Snowstorm (aka MultiSim) by running a series of short trays, each with a different ice model. This works by front-loading as much of the expensive initialization (reading the GCD file, setting up PROPOSAL/Geant4, etc) as possible, so that only the propagation kernel needs to be recompiled for every tray. """ # instantiate baseline detector setup. # this will help construct the baseline characteristics before applying # the perturbers print("Setting up detector... ", end="") clsimParams = setupDetector( GCDFile=cfg['gcd'], SimulateFlashers=bool(cfg['FlasherInfoVectName'] or cfg['FlasherPulseSeriesName']), IceModelLocation=ice_model_location, DisableTilt=cfg['DisableTilt'], UnWeightedPhotons=cfg['UnWeightedPhotons'], UnWeightedPhotonsScalingFactor=cfg['UnWeightedPhotonsScalingFactor'], UseI3PropagatorService=cfg['UseI3PropagatorService'], UseGeant4=cfg['UseGeant4'], CrossoverEnergyEM=cfg['CrossoverEnergyEM'], CrossoverEnergyHadron=cfg['CrossoverEnergyHadron'], UseCascadeExtension=cfg['UseCascadeExtension'], StopDetectedPhotons=cfg['StopDetectedPhotons'], DOMOversizeFactor=cfg['DOMOversizeFactor'], UnshadowedFraction=cfg['UnshadowedFraction'], HoleIceParameterization=hole_ice_parameterization, WavelengthAcceptance=cfg['WavelengthAcceptance'], DOMRadius=cfg['DOMRadius'], CableOrientation=cfg['CableOrientation'], IgnoreSubdetectors=cfg['IgnoreSubdetectors'], ) print("done") print("Setting up OpenCLDevices... ", end="") openCLDevices = configureOpenCLDevices( UseGPUs=cfg['UseGPUs'], UseCPUs=not cfg['UseGPUs'], OverrideApproximateNumberOfWorkItems=cfg[ 'OverrideApproximateNumberOfWorkItems'], DoNotParallelize=cfg['DoNotParallelize'], UseOnlyDeviceNumber=cfg['UseOnlyDeviceNumber']) print("done") # ------------------- # Setup perturbations # ------------------- # create empty "perturber" object perturber = Perturber() # get perturbation_cfg dict to simplify calls perturbation_cfg = snowstorm_config["Perturbations"] # loop over all perturbations in the perturbation_cfg print("Setting up perturbers... ") for name, params in perturbation_cfg.items(): # catch special case of IceWavePlusModes if name == "IceWavePlusModes": if not params["apply"]: continue if params["type"] == "default": print("-> adding {} of type {}".format(name, params["type"])) perturber.add('IceWavePlusModes', *icewave.get_default_perturbation()) continue elif hasattr(snowstorm_perturbers, params["type"]): print("-> adding {} of type {}".format(name, params["type"])) get_perturber = getattr(snowstorm_perturbers, params["type"]) perturber.add('IceWavePlusModes', *get_perturber(**params['settings'])) continue else: msg = "IceWavePlusModes of type '{}' are not implemented(yet)." raise NotImplementedError(msg.format(params["type"])) # all other cases if params["type"] == "delta": print("-> adding {} of type {}".format(name, params["type"])) params = params["delta"] perturber.add(name, all_parametrizations[name], DeltaDistribution(params["x0"])) elif params["type"] == "gauss": print("-> adding {} of type {}".format(name, params["type"])) params = params["gauss"] # Caution: MultivariateNormal expect the covariance matrix as # first argument, so we need to use sigma**2 perturber.add(name, all_parametrizations[name], MultivariateNormal( dataclasses.I3Matrix(np.diag(params["sigma"])**2), params["mu"])) elif params["type"] == "uniform": print("-> adding {} of type {}".format(name, params["type"])) params = params["uniform"] perturber.add(name, all_parametrizations[name], UniformDistribution( [dataclasses.make_pair(*limits) for limits in params["limits"]])) else: msg = "Perturbation '{}' of type '{}' not implemented." raise NotImplementedError(msg.format(name, params["type"])) print("done") # Setting up some other things gcdFrames = list(dataio.I3File(cfg['gcd'])) inputStream = dataio.I3FrameSequence([infile]) summary = dataclasses.I3MapStringDouble() intermediateOutputFiles = [] # -------------- # Run PhotonProp # -------------- # start a model counter model_counter = 0 # Execute photon propagation print("Executing photon propagation...", end="") while inputStream.more(): # measure CLSimInit time time_CLSimInit_start = time.time() tray = I3Tray() tray.context['I3RandomService'] = random_service tray.context['I3SummaryService'] = summary # make a mutable copy of the config dict config = dict(clsimParams) # populate the M frame with I3FrameObjects from clsimParams model = icetray.I3Frame('M') for k, v in config.items(): if isinstance(v, icetray.I3FrameObject): model[k] = v # apply perturbations in the order they were configured perturber.perturb(random_service, model) # check for items in the M-frame that were changed/added # by the perturbers for k in model.keys(): if k.startswith('Snowstorm'): # keep all Snowstorm keys continue if k not in config: msg = "\n {} was put in the M frame, but does not appear in " msg += "the CLSim configuration dict" raise KeyError(msg.format(k)) if config[k] != model[k]: # if an items was changed, copy it back to clsimParams config[k] = model[k] else: # remove unmodified items from the M frame del model[k] # add "persistent" I3Reader tray.Add(FrameSequenceReader, Sequence=itertools.chain(gcdFrames, [model], inputStream)) # inject an S frame if it doesn't exist tray.Add(EnsureSFrame, Enable=len(intermediateOutputFiles) == 0) # write pertubations to frame def populate_s_frame(frame): perturber.to_frame(frame) tray.Add(populate_s_frame, Streams=[icetray.I3Frame.Stream('S')]) # Add Bumper to stop the tray after NumEventsPerModel Q-frames tray.Add(Bumper, NumFrames=cfg['NumEventsPerModel']) # initialize CLSim server and setup the propagators server_location = tempfile.mkstemp(prefix='clsim-server-')[1] address = 'ipc://'+server_location converters = setupPropagators( random_service, config, UseGPUs=cfg['UseGPUs'], UseCPUs=not cfg['UseGPUs'], OverrideApproximateNumberOfWorkItems=cfg[ 'OverrideApproximateNumberOfWorkItems'], DoNotParallelize=cfg['DoNotParallelize'], UseOnlyDeviceNumber=cfg['UseOnlyDeviceNumber'] ) server = clsim.I3CLSimServer( address, clsim.I3CLSimStepToPhotonConverterSeries(converters)) # stash server instance in the context to keep it alive tray.context['CLSimServer'] = server # recycle StepGenerator to prevent repeated, expensive initialization if 'StepGenerator' in cfg['ExtraArgumentsToI3CLSimClientModule']: stepGenerator = \ cfg['ExtraArgumentsToI3CLSimClientModule']['StepGenerator'] stepGenerator.SetMediumProperties(config['MediumProperties']) stepGenerator.SetWlenBias(config['WavelengthGenerationBias']) # add CLSim server to tray module_config = \ tray.Add( I3CLSimMakePhotonsWithServer, ServerAddress=address, DetectorSettings=config, MCTreeName=cfg['MCTreeName'], OutputMCTreeName=cfg['OutputMCTreeName'], FlasherInfoVectName=cfg['FlasherInfoVectName'], FlasherPulseSeriesName=cfg['FlasherPulseSeriesName'], PhotonSeriesName=cfg['PhotonSeriesName'], MCPESeriesName=cfg['MCPESeriesName'], RandomService=random_service, ParticleHistory=cfg['ParticleHistory'], ParticleHistoryGranularity=cfg['ParticleHistoryGranularity'], ExtraArgumentsToI3CLSimClientModule=cfg[ 'ExtraArgumentsToI3CLSimClientModule'], ) # recycle StepGenerator to prevent repeated, expensive initialization cfg['ExtraArgumentsToI3CLSimClientModule']['StepGenerator'] = \ module_config['StepGenerator'] # write to temporary output file intermediateOutputFiles.append( tempfile.mkstemp(suffix=(outfile.split("/"))[-1])[1]) tray.Add("I3Writer", Filename=intermediateOutputFiles[-1], DropOrphanStreams=[icetray.I3Frame.TrayInfo], Streams=[icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation, icetray.I3Frame.Stream('M'), icetray.I3Frame.Stream('m'), icetray.I3Frame.DAQ, icetray.I3Frame.Physics]) # gather statistics in the "I3SummaryService" tray.Add(GatherStatistics) # measure CLSimInit time time_CLSimInit = time.time() - time_CLSimInit_start summary["CLSimInitTime_{:03d}".format(model_counter)] = time_CLSimInit if "TotalCLSimInitTime" not in summary: summary["TotalCLSimInitTime"] = time_CLSimInit else: summary["TotalCLSimInitTime"] += time_CLSimInit # measure CLSimTray time time_CLSimTray_start = time.time() # Execute Tray tray.Execute() # measure CLSimTray time time_CLSimTray = time.time() - time_CLSimTray_start summary["CLSimTrayTime_{:03d}".format(model_counter)] = time_CLSimTray if "TotalCLSimTrayTime" not in summary: summary["TotalCLSimTrayTime"] = time_CLSimTray else: summary["TotalCLSimTrayTime"] += time_CLSimTray # remove the temp file made by the server location thingy os.unlink(server_location) # increase model counter model_counter += 1 print("done") # Add number of models to summary summary["TotalNumberOfModels"] = model_counter # Concatenate intermediate files print("Concatenating temporary files... ", end='') tray = I3Tray() tray.Add(dataio.I3Reader, "I3Reader", FilenameList=intermediateOutputFiles) tray.Add("I3Writer", Filename=outfile, DropOrphanStreams=[icetray.I3Frame.TrayInfo], Streams=[icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation, icetray.I3Frame.Stream('M'), icetray.I3Frame.Stream('m'), icetray.I3Frame.DAQ, icetray.I3Frame.Physics]) tray.Execute() tray.Finish() print("done") print("Cleaning up Temporary files... ") for fname in intermediateOutputFiles: os.unlink(fname) print("done") # Recalculate averages print("Writing summary file... ", end='') if cfg['UseGPUs']: if summary['TotalHostTime'] > 0.0: summary['DeviceUtilization'] = \ summary['TotalDeviceTime']/summary['TotalHostTime'] if summary['TotalNumPhotonsGenerated'] > 0.0: summary['AverageDeviceTimePerPhoton'] = \ summary['TotalDeviceTime']/summary['TotalNumPhotonsGenerated'] if summary['TotalNumPhotonsGenerated'] > 0.0: summary['AverageHostTimePerPhoton'] = \ summary['TotalHostTime']/summary['TotalNumPhotonsGenerated'] if cfg['SummaryFile']: with open(cfg['SummaryFile'], 'w') as f: yaml.dump(dict(summary), f) print("done") print('--------') print('Summary:') print('--------') for key, value in summary.items(): print('\t{}: {}'.format(key, value)) print('--------\n') # Hurray! print("All finished!") # say something about the runtime end_time = time.time() print("That took "+str(end_time - start_time)+" seconds.")
feed() t.join() icetray.logging.log_info("base {} done".format(base), unit='clsim.testCLSimServer') # icetray.logging.set_level_for_unit('I3CLSimServer', 'TRACE') # icetray.logging.set_level_for_unit('I3CLSimServer', 'DEBUG') icetray.logging.set_level_for_unit('I3CLSimClient', 'TRACE') icetray.logging.set_level_for_unit('clsim.testCLSimServer', 'TRACE') # First, ensure that the test passes when the converter is called directly test_client(DummyConverter(), 10) # Now, call through the server in a separate process converters = clsim.I3CLSimStepToPhotonConverterSeries([DummyConverter()]) server = clsim.I3CLSimServer('tcp://127.0.0.1:*',converters) address = server.GetAddress() def fire_a_few(num_steps=10, base=0): # NB: the Python logging bridge deadlocks from secondary threads in Py3 if sys.version_info.major == 2: icetray.logging.BASIC_FORMAT = "{} %(filename)s:%(lineno)s %(levelname)s: %(message)s".format(base) icetray.logging.console() icetray.logging.set_level_for_unit('clsim.testCLSimServer', 'TRACE') icetray.logging.set_level_for_unit('I3CLSimClient', 'TRACE') icetray.logging.log_debug("client {} connecting to {}".format(base, address), unit='clsim.testCLSimServer') client = clsim.I3CLSimClient(address) icetray.logging.log_debug("client {} connected".format(base), unit='clsim.testCLSimServer') testing.assert_equal( client.workgroupSize, 8 ) testing.assert_equal( client.maxNumWorkitems, 64 )