def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: cfg = yaml.load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) infile = cfg['infile_pattern'].format(**cfg) infile = infile.replace(' ', '0') tray = I3Tray() tray.context['I3FileStager'] = dataio.get_stagers() random_service, _, run_id = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed']) tray.context['I3RandomService'] = random_service tray.Add('I3Reader', FilenameList=[cfg['gcd_pass2'], infile]) if run_number < cfg['det_pass2_keep_all_upto']: cfg['det_keep_mc_hits'] = True cfg['det_keep_propagated_mc_tree'] = True cfg['det_keep_mc_pulses'] = True tray.AddSegment(segments.DetectorSim, "Detector5Sim", RandomService='I3RandomService', RunID=run_id, GCDFile=cfg['gcd_pass2'], KeepMCHits=cfg['det_keep_mc_hits'], KeepPropagatedMCTree=cfg['det_keep_propagated_mc_tree'], KeepMCPulses=cfg['det_keep_mc_pulses'], SkipNoiseGenerator=cfg['det_skip_noise_generation'], LowMem=cfg['det_low_mem'], InputPESeriesMapName=MCPE_SERIES_MAP, BeaconLaunches=cfg['det_add_beacon_launches'], FilterTrigger=cfg['det_filter_trigger']) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace(' ', '0') print(outfile) print(cfg['outfile_pattern']) tray.AddModule("I3Writer", "EventWriter", filename=outfile, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation ]) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: if int(yaml.__version__[0]) < 5: # backwards compatibility for yaml versions before version 5 cfg = yaml.load(stream) else: cfg = yaml.full_load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace(' ', '0') click.echo('Run: {}'.format(run_number)) click.echo('ParticleType: {}'.format(cfg['particle_type'])) click.echo('Outfile: {}'.format(outfile)) click.echo('n_events_per_run: {}'.format(cfg['n_events_per_run'])) click.echo('smearing_angle: {}'.format(cfg['smearing_angle'])) click.echo('skymap_path: {}'.format(cfg['skymap_path'])) tray = I3Tray() random_services, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=2) tray.AddModule( 'I3InfiniteSource', 'source', # Prefix=gcdfile, Stream=icetray.I3Frame.DAQ) tray.AddModule(ParticleFactory, 'make_particles', particle_type=cfg['particle_type'], map_filename=cfg['skymap_path'], num_events=cfg['n_events_per_run'], smearing_angle=cfg['smearing_angle'] * I3Units.deg, random_state=cfg['seed'], random_service=random_services[0]) tray.AddSegment(segments.PropagateMuons, 'propagate_muons', RandomService=random_services[1], InputMCTreeName='I3MCTree', **cfg['muon_propagation_config']) tray.AddModule('I3Writer', 'write', Filename=outfile, Streams=[icetray.I3Frame.DAQ, icetray.I3Frame.Stream('M')]) tray.AddModule('TrashCan', 'trash') tray.Execute() tray.Finish() del tray
def run_snowstorm_propagation(cfg, infile, outfile): """Run SnowStorm Propagation. Adopted from: https://code.icecube.wisc.edu/projects/icecube/browser/IceCube/ meta-projects/combo/stable/simprod-scripts/resources/scripts/ SnowSuite/3-Snowstorm.py Parameters ---------- cfg : dict Dictionary with configuration settings. infile : str Path to input file. outfile : str Path to output file. """ start_time = time.time() # -------- # Settings # -------- default_args = { # required 'NumEventsPerModel': 100, 'DOMOversizeFactor': 1., 'UseI3PropagatorService': True, # optional 'UseGPUs': True, 'SummaryFile': 'summary_snowstorm.yaml', 'UseOnlyDeviceNumber': None, 'MCTreeName': 'I3MCTree', 'OutputMCTreeName': None, 'FlasherInfoVectName': None, 'FlasherPulseSeriesName': None, 'PhotonSeriesName': None, 'MCPESeriesName': "I3MCPESeriesMap", 'DisableTilt': False, 'UnWeightedPhotons': False, 'UnWeightedPhotonsScalingFactor': None, 'UseGeant4': False, 'ParticleHistory': True, 'ParticleHistoryGranularity': 1*icetray.I3Units.m, 'CrossoverEnergyEM': None, 'CrossoverEnergyHadron': None, 'UseCascadeExtension': True, 'StopDetectedPhotons': True, 'PhotonHistoryEntries': 0, 'DoNotParallelize': False, 'UnshadowedFraction': 1.0, 'WavelengthAcceptance': None, 'DOMRadius': 0.16510*icetray.I3Units.m, 'CableOrientation': None, 'OverrideApproximateNumberOfWorkItems': None, 'IgnoreSubdetectors': ["IceTop"], 'ExtraArgumentsToI3CLSimClientModule': dict(), } # overwrite default settings default_args.update(cfg) cfg = default_args snowstorm_config = cfg['snowstorm_config'] if cfg['SummaryFile'] is not None: cfg['SummaryFile'] = cfg['SummaryFile'].format(**cfg) ice_model_location = \ os.path.expandvars(snowstorm_config["IceModelLocation"]) hole_ice_parameterization = \ os.path.expandvars(snowstorm_config["HoleIceParameterization"]) # set units to meter cfg['ParticleHistoryGranularity'] *= icetray.I3Units.m cfg['DOMRadius'] *= icetray.I3Units.m # Print out most important settings click.echo('\n---------------') click.echo('Script Settigns') click.echo('---------------') click.echo('\tInput: {}'.format(infile)) click.echo('\tGCDFile: {}'.format(cfg['gcd'])) click.echo('\tOutput: {}'.format(outfile)) for key in ['DOMOversizeFactor', 'UseI3PropagatorService', 'UseGPUs', 'SummaryFile']: click.echo('\t{}: {}'.format(key, cfg[key])) click.echo('---------------\n') # get random service random_services, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=1, use_gslrng=cfg['random_service_use_gslrng']) random_service = random_services[0] """ Setup and run Snowstorm (aka MultiSim) by running a series of short trays, each with a different ice model. This works by front-loading as much of the expensive initialization (reading the GCD file, setting up PROPOSAL/Geant4, etc) as possible, so that only the propagation kernel needs to be recompiled for every tray. """ # instantiate baseline detector setup. # this will help construct the baseline characteristics before applying # the perturbers print("Setting up detector... ", end="") clsimParams = setupDetector( GCDFile=cfg['gcd'], SimulateFlashers=bool(cfg['FlasherInfoVectName'] or cfg['FlasherPulseSeriesName']), IceModelLocation=ice_model_location, DisableTilt=cfg['DisableTilt'], UnWeightedPhotons=cfg['UnWeightedPhotons'], UnWeightedPhotonsScalingFactor=cfg['UnWeightedPhotonsScalingFactor'], UseI3PropagatorService=cfg['UseI3PropagatorService'], UseGeant4=cfg['UseGeant4'], CrossoverEnergyEM=cfg['CrossoverEnergyEM'], CrossoverEnergyHadron=cfg['CrossoverEnergyHadron'], UseCascadeExtension=cfg['UseCascadeExtension'], StopDetectedPhotons=cfg['StopDetectedPhotons'], DOMOversizeFactor=cfg['DOMOversizeFactor'], UnshadowedFraction=cfg['UnshadowedFraction'], HoleIceParameterization=hole_ice_parameterization, WavelengthAcceptance=cfg['WavelengthAcceptance'], DOMRadius=cfg['DOMRadius'], CableOrientation=cfg['CableOrientation'], IgnoreSubdetectors=cfg['IgnoreSubdetectors'], ) print("done") print("Setting up OpenCLDevices... ", end="") openCLDevices = configureOpenCLDevices( UseGPUs=cfg['UseGPUs'], UseCPUs=not cfg['UseGPUs'], OverrideApproximateNumberOfWorkItems=cfg[ 'OverrideApproximateNumberOfWorkItems'], DoNotParallelize=cfg['DoNotParallelize'], UseOnlyDeviceNumber=cfg['UseOnlyDeviceNumber']) print("done") # ------------------- # Setup perturbations # ------------------- # create empty "perturber" object perturber = Perturber() # get perturbation_cfg dict to simplify calls perturbation_cfg = snowstorm_config["Perturbations"] # loop over all perturbations in the perturbation_cfg print("Setting up perturbers... ") for name, params in perturbation_cfg.items(): # catch special case of IceWavePlusModes if name == "IceWavePlusModes": if not params["apply"]: continue if params["type"] == "default": print("-> adding {} of type {}".format(name, params["type"])) perturber.add('IceWavePlusModes', *icewave.get_default_perturbation()) continue elif hasattr(snowstorm_perturbers, params["type"]): print("-> adding {} of type {}".format(name, params["type"])) get_perturber = getattr(snowstorm_perturbers, params["type"]) perturber.add('IceWavePlusModes', *get_perturber(**params['settings'])) continue else: msg = "IceWavePlusModes of type '{}' are not implemented(yet)." raise NotImplementedError(msg.format(params["type"])) # all other cases if params["type"] == "delta": print("-> adding {} of type {}".format(name, params["type"])) params = params["delta"] perturber.add(name, all_parametrizations[name], DeltaDistribution(params["x0"])) elif params["type"] == "gauss": print("-> adding {} of type {}".format(name, params["type"])) params = params["gauss"] # Caution: MultivariateNormal expect the covariance matrix as # first argument, so we need to use sigma**2 perturber.add(name, all_parametrizations[name], MultivariateNormal( dataclasses.I3Matrix(np.diag(params["sigma"])**2), params["mu"])) elif params["type"] == "uniform": print("-> adding {} of type {}".format(name, params["type"])) params = params["uniform"] perturber.add(name, all_parametrizations[name], UniformDistribution( [dataclasses.make_pair(*limits) for limits in params["limits"]])) else: msg = "Perturbation '{}' of type '{}' not implemented." raise NotImplementedError(msg.format(name, params["type"])) print("done") # Setting up some other things gcdFrames = list(dataio.I3File(cfg['gcd'])) inputStream = dataio.I3FrameSequence([infile]) summary = dataclasses.I3MapStringDouble() intermediateOutputFiles = [] # -------------- # Run PhotonProp # -------------- # start a model counter model_counter = 0 # Execute photon propagation print("Executing photon propagation...", end="") while inputStream.more(): # measure CLSimInit time time_CLSimInit_start = time.time() tray = I3Tray() tray.context['I3RandomService'] = random_service tray.context['I3SummaryService'] = summary # make a mutable copy of the config dict config = dict(clsimParams) # populate the M frame with I3FrameObjects from clsimParams model = icetray.I3Frame('M') for k, v in config.items(): if isinstance(v, icetray.I3FrameObject): model[k] = v # apply perturbations in the order they were configured perturber.perturb(random_service, model) # check for items in the M-frame that were changed/added # by the perturbers for k in model.keys(): if k.startswith('Snowstorm'): # keep all Snowstorm keys continue if k not in config: msg = "\n {} was put in the M frame, but does not appear in " msg += "the CLSim configuration dict" raise KeyError(msg.format(k)) if config[k] != model[k]: # if an items was changed, copy it back to clsimParams config[k] = model[k] else: # remove unmodified items from the M frame del model[k] # add "persistent" I3Reader tray.Add(FrameSequenceReader, Sequence=itertools.chain(gcdFrames, [model], inputStream)) # inject an S frame if it doesn't exist tray.Add(EnsureSFrame, Enable=len(intermediateOutputFiles) == 0) # write pertubations to frame def populate_s_frame(frame): perturber.to_frame(frame) tray.Add(populate_s_frame, Streams=[icetray.I3Frame.Stream('S')]) # Add Bumper to stop the tray after NumEventsPerModel Q-frames tray.Add(Bumper, NumFrames=cfg['NumEventsPerModel']) # initialize CLSim server and setup the propagators server_location = tempfile.mkstemp(prefix='clsim-server-')[1] address = 'ipc://'+server_location converters = setupPropagators( random_service, config, UseGPUs=cfg['UseGPUs'], UseCPUs=not cfg['UseGPUs'], OverrideApproximateNumberOfWorkItems=cfg[ 'OverrideApproximateNumberOfWorkItems'], DoNotParallelize=cfg['DoNotParallelize'], UseOnlyDeviceNumber=cfg['UseOnlyDeviceNumber'] ) server = clsim.I3CLSimServer( address, clsim.I3CLSimStepToPhotonConverterSeries(converters)) # stash server instance in the context to keep it alive tray.context['CLSimServer'] = server # recycle StepGenerator to prevent repeated, expensive initialization if 'StepGenerator' in cfg['ExtraArgumentsToI3CLSimClientModule']: stepGenerator = \ cfg['ExtraArgumentsToI3CLSimClientModule']['StepGenerator'] stepGenerator.SetMediumProperties(config['MediumProperties']) stepGenerator.SetWlenBias(config['WavelengthGenerationBias']) # add CLSim server to tray module_config = \ tray.Add( I3CLSimMakePhotonsWithServer, ServerAddress=address, DetectorSettings=config, MCTreeName=cfg['MCTreeName'], OutputMCTreeName=cfg['OutputMCTreeName'], FlasherInfoVectName=cfg['FlasherInfoVectName'], FlasherPulseSeriesName=cfg['FlasherPulseSeriesName'], PhotonSeriesName=cfg['PhotonSeriesName'], MCPESeriesName=cfg['MCPESeriesName'], RandomService=random_service, ParticleHistory=cfg['ParticleHistory'], ParticleHistoryGranularity=cfg['ParticleHistoryGranularity'], ExtraArgumentsToI3CLSimClientModule=cfg[ 'ExtraArgumentsToI3CLSimClientModule'], ) # recycle StepGenerator to prevent repeated, expensive initialization cfg['ExtraArgumentsToI3CLSimClientModule']['StepGenerator'] = \ module_config['StepGenerator'] # write to temporary output file intermediateOutputFiles.append( tempfile.mkstemp(suffix=(outfile.split("/"))[-1])[1]) tray.Add("I3Writer", Filename=intermediateOutputFiles[-1], DropOrphanStreams=[icetray.I3Frame.TrayInfo], Streams=[icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation, icetray.I3Frame.Stream('M'), icetray.I3Frame.Stream('m'), icetray.I3Frame.DAQ, icetray.I3Frame.Physics]) # gather statistics in the "I3SummaryService" tray.Add(GatherStatistics) # measure CLSimInit time time_CLSimInit = time.time() - time_CLSimInit_start summary["CLSimInitTime_{:03d}".format(model_counter)] = time_CLSimInit if "TotalCLSimInitTime" not in summary: summary["TotalCLSimInitTime"] = time_CLSimInit else: summary["TotalCLSimInitTime"] += time_CLSimInit # measure CLSimTray time time_CLSimTray_start = time.time() # Execute Tray tray.Execute() # measure CLSimTray time time_CLSimTray = time.time() - time_CLSimTray_start summary["CLSimTrayTime_{:03d}".format(model_counter)] = time_CLSimTray if "TotalCLSimTrayTime" not in summary: summary["TotalCLSimTrayTime"] = time_CLSimTray else: summary["TotalCLSimTrayTime"] += time_CLSimTray # remove the temp file made by the server location thingy os.unlink(server_location) # increase model counter model_counter += 1 print("done") # Add number of models to summary summary["TotalNumberOfModels"] = model_counter # Concatenate intermediate files print("Concatenating temporary files... ", end='') tray = I3Tray() tray.Add(dataio.I3Reader, "I3Reader", FilenameList=intermediateOutputFiles) tray.Add("I3Writer", Filename=outfile, DropOrphanStreams=[icetray.I3Frame.TrayInfo], Streams=[icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation, icetray.I3Frame.Stream('M'), icetray.I3Frame.Stream('m'), icetray.I3Frame.DAQ, icetray.I3Frame.Physics]) tray.Execute() tray.Finish() print("done") print("Cleaning up Temporary files... ") for fname in intermediateOutputFiles: os.unlink(fname) print("done") # Recalculate averages print("Writing summary file... ", end='') if cfg['UseGPUs']: if summary['TotalHostTime'] > 0.0: summary['DeviceUtilization'] = \ summary['TotalDeviceTime']/summary['TotalHostTime'] if summary['TotalNumPhotonsGenerated'] > 0.0: summary['AverageDeviceTimePerPhoton'] = \ summary['TotalDeviceTime']/summary['TotalNumPhotonsGenerated'] if summary['TotalNumPhotonsGenerated'] > 0.0: summary['AverageHostTimePerPhoton'] = \ summary['TotalHostTime']/summary['TotalNumPhotonsGenerated'] if cfg['SummaryFile']: with open(cfg['SummaryFile'], 'w') as f: yaml.dump(dict(summary), f) print("done") print('--------') print('Summary:') print('--------') for key, value in summary.items(): print('\t{}: {}'.format(key, value)) print('--------\n') # Hurray! print("All finished!") # say something about the runtime end_time = time.time() print("That took "+str(end_time - start_time)+" seconds.")
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: if int(yaml.__version__[0]) < 5: # backwards compatibility for yaml versions before version 5 cfg = yaml.load(stream) else: cfg = yaml.full_load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) infile = cfg['infile_pattern'].format(**cfg) infile = infile.replace(' ', '0') infile = infile.replace('Level0.{}'.format(cfg['previous_step']), 'Level0.{}'.format(cfg['previous_step'] % 10)) infile = infile.replace('2012_pass2', 'pass2') if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace('Level0.{}'.format(cfg['step']), 'Level0.{}'.format(cfg['step'] % 10)) outfile = outfile.replace(' ', '0') outfile = outfile.replace('2012_pass2', 'pass2') print('Outfile != $FINAL_OUT clean up for crashed scripts not possible!') # collect keys which are to be kept in a addition to the official keep keys # of the standard L1 and L2 processing if 'additional_keep_keys' in cfg: additional_keep_keys = cfg['additional_keep_keys'] + muongun_keys else: additional_keep_keys = muongun_keys additional_keep_keys += [ 'BiasedMuonWeighter', 'BiasedMuonCorridorWeighter', 'BiasedMESCHotspotWeighter', 'BiasedSimulationWeight', 'PROPOSALStorm', 'PROPOSALStormUniformRanges', 'MCVetoMuonInjectionInfo', 'MMCTrackListVetoMuon', 'CombinedMuonVetoI3MCTree', 'I3MCTreeVetoMuon', 'I3MCTreeVetoMuon_preMuonProp', 'I3MCTreeVetoMuon_preMuonProp_RNGState', ] tray = I3Tray() """The main L1 script""" tray.AddModule('I3Reader', 'i3 reader', FilenameList=[cfg['gcd_pass2'], infile]) # run online filters online_kwargs = {} if SPLINE_TABLES: online_kwargs.update({ 'SplineRecoAmplitudeTable': os.path.join( SPLINE_TABLES, 'InfBareMu_mie_abs_z20a10.fits'), 'SplineRecoTimingTable': os.path.join( SPLINE_TABLES, 'InfBareMu_mie_prob_z20a10.fits'), # 'alert_followup_base_GCD_filename': cfg['gcd_pass2'], }) if cfg['L1_pass2_run_gfu'] is not None: online_kwargs['gfu_enabled'] = cfg['L1_pass2_run_gfu'] if 'L1_needs_wavedeform_spe_corr' not in cfg: cfg['L1_needs_wavedeform_spe_corr'] = False tray.AddSegment(OnlineFilter, "OnlineFilter", decode=False, simulation=True, vemcal_enabled=False, alert_followup=False, needs_wavedeform_spe_corr=cfg[ 'L1_needs_wavedeform_spe_corr'], **online_kwargs ) # make random service _, seed = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=0) filter_mask_randoms = phys_services.I3GSLRandomService(seed) # override MinBias Prescale filterconfigs = filter_globals.filter_pairs + filter_globals.sdst_pairs print(cfg['L1_min_bias_prescale']) if cfg['L1_min_bias_prescale']: for i, filtertuple in enumerate(filterconfigs): if filtertuple[0] == filter_globals.FilterMinBias: del filterconfigs[i] filterconfigs.append((filtertuple[0], cfg['L1_min_bias_prescale'])) break print(filterconfigs) # Generate filter Masks for all P frames tray.AddModule(filter_tools.FilterMaskMaker, "MakeFilterMasks", OutputMaskName=filter_globals.filter_mask, FilterConfigs=filterconfigs, RandomService=filter_mask_randoms) # Merge the FilterMasks tray.AddModule("OrPframeFilterMasks", "make_q_filtermask", InputName=filter_globals.filter_mask, OutputName=filter_globals.qfilter_mask) # Q+P frame specific keep module needs to go first, as KeepFromSubstram # will rename things, let's rename post keep. def is_Q(frame): return frame.Stop == frame.DAQ simulation_keeps = [ 'BackgroundI3MCTree', 'BackgroundI3MCTreePEcounts', 'BackgroundI3MCPESeriesMap', 'BackgroundI3MCTree_preMuonProp', 'BackgroundI3MCTree_preMuonProp_RNGState', 'BackgroundMMCTrackList', 'BeaconLaunches', 'CorsikaInteractionHeight', 'CorsikaWeightMap', 'EventProperties', 'GenerationSpec', 'I3LinearizedMCTree', 'I3MCTree', 'I3MCTreePEcounts', 'I3MCTree_preMuonProp', 'I3MCTree_preMuonProp_RNGState', 'I3MCPESeriesMap', 'I3MCPESeriesMapWithoutNoise', 'I3MCPESeriesMapParticleIDMap', 'I3MCPulseSeriesMap', 'I3MCPulseSeriesMapParticleIDMap', 'I3MCPulseSeriesMapPrimaryIDMap', 'I3MCWeightDict', 'LeptonInjectorProperties', 'MCHitSeriesMap', 'MCPrimary', 'MCPrimaryInfo', 'MMCTrackList', 'PolyplopiaInfo', 'PolyplopiaPrimary', 'RNGState', 'SignalI3MCPEs', 'SimTrimmer', # for SimTrimmer flag 'TimeShift', # the time shift amount 'WIMP_params', # Wimp-sim 'noise_weight', # weights for noise-only vuvuzela simulations 'I3GENIEResultDict' # weight informaition for GENIE simulations ] + additional_keep_keys keep_before_merge = filter_globals.q_frame_keeps + [ 'InIceDSTPulses', # keep DST pulse masks 'IceTopDSTPulses', 'CalibratedWaveformRange', # keep calibration info 'UncleanedInIcePulsesTimeRange', 'SplitUncleanedInIcePulses', 'SplitUncleanedInIcePulsesTimeRange', 'SplitUncleanedInIceDSTPulsesTimeRange', 'CalibrationErrata', 'SaturationWindows', 'InIceRawData', # keep raw data for now 'IceTopRawData', ] + simulation_keeps tray.AddModule("Keep", "keep_before_merge", keys=keep_before_merge, If=is_Q) # second set of prekeeps, conditional on filter content, based on newly # created Qfiltermask # Determine if we should apply harsh keep for events that failed to pass # any filter # Note: excluding the sdst_streams entries tray.AddModule("I3IcePickModule<FilterMaskFilter>", "filterMaskCheckAll", FilterNameList=filter_globals.filter_streams, FilterResultName=filter_globals.qfilter_mask, DecisionName="PassedAnyFilter", DiscardEvents=False, Streams=[icetray.I3Frame.DAQ]) def do_save_just_superdst(frame): if frame.Has("PassedAnyFilter"): if not frame["PassedAnyFilter"].value: return True # <- Event failed to pass any filter. else: return False # <- Event passed some filter else: icetray.logging.log_error("Failed to find key frame Bool!!") return False keep_only_superdsts = filter_globals.keep_nofilterpass + [ 'PassedAnyFilter', 'InIceDSTPulses', 'IceTopDSTPulses', 'SplitUncleanedInIcePulses', 'SplitUncleanedInIcePulsesTimeRange', 'SplitUncleanedInIceDSTPulsesTimeRange', 'RNGState'] + simulation_keeps tray.AddModule("Keep", "KeepOnlySuperDSTs", keys=keep_only_superdsts, If=do_save_just_superdst) # Now clean up the events that not even the SuperDST filters passed on tray.AddModule("I3IcePickModule<FilterMaskFilter>", "filterMaskCheckSDST", FilterNameList=filter_globals.sdst_streams, FilterResultName=filter_globals.qfilter_mask, DecisionName="PassedKeepSuperDSTOnly", DiscardEvents=False, Streams=[icetray.I3Frame.DAQ]) def dont_save_superdst(frame): if frame.Has("PassedKeepSuperDSTOnly") and \ frame.Has("PassedAnyFilter"): if frame["PassedAnyFilter"].value: return False # <- these passed a regular filter, keeper elif not frame["PassedKeepSuperDSTOnly"].value: return True # <- Event failed to pass SDST filter. else: return False # <- Event passed some SDST filter else: icetray.logging.log_error("Failed to find key frame Bool!!") return False # backward compatibility if 'L1_keep_untriggered' in cfg and cfg['L1_keep_untriggered']: discard_substream_and_keys = False else: discard_substream_and_keys = True if discard_substream_and_keys: tray.AddModule( "Keep", "KeepOnlyDSTs", keys=filter_globals.keep_dst_only + [ "PassedAnyFilter", "PassedKeepSuperDSTOnly", filter_globals.eventheader] + additional_keep_keys, If=dont_save_superdst) # Frames should now contain only what is needed. now flatten, # write/send to server # Squish P frames back to single Q frame, one for each split: tray.AddModule("KeepFromSubstream", "null_stream", StreamName=filter_globals.NullSplitter, KeepKeys=filter_globals.null_split_keeps) in_ice_keeps = filter_globals.inice_split_keeps + \ filter_globals.onlinel2filter_keeps in_ice_keeps = in_ice_keeps + [ 'I3EventHeader', 'SplitUncleanedInIcePulses', 'SplitUncleanedInIcePulsesTimeRange', 'TriggerSplitterLaunchWindow', 'I3TriggerHierarchy', 'GCFilter_GCFilterMJD'] + additional_keep_keys tray.AddModule("Keep", "inice_keeps", keys=in_ice_keeps, If=which_split(split_name=filter_globals.InIceSplitter),) tray.AddModule("KeepFromSubstream", "icetop_split_stream", StreamName=filter_globals.IceTopSplitter, KeepKeys=filter_globals.icetop_split_keeps) # Apply small keep list (SuperDST/SmallTrig/DST/FilterMask for non-filter # passers # Remove I3DAQData object for events not passing one of the # 'filters_keeping_allraw' tray.AddModule("I3IcePickModule<FilterMaskFilter>", "filterMaskCheck", FilterNameList=filter_globals.filters_keeping_allraw, FilterResultName=filter_globals.qfilter_mask, DecisionName="PassedConventional", DiscardEvents=False, Streams=[icetray.I3Frame.DAQ]) # Clean out the Raw Data when not passing conventional filter def I3RawDataCleaner(frame): if not (('PassedConventional' in frame and frame['PassedConventional'].value == True) or ('SimTrimmer' in frame and frame['SimTrimmer'].value == True)): frame.Delete('InIceRawData') frame.Delete('IceTopRawData') tray.AddModule(I3RawDataCleaner, "CleanErrataForConventional", Streams=[icetray.I3Frame.DAQ]) tray.AddModule("I3Writer", "EventWriter", filename=outfile, Streams=[icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation, icetray.I3Frame.Stream('m'), icetray.I3Frame.Stream('M')]) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: cfg = yaml.full_load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace(' ', '0') click.echo('Run: {}'.format(run_number)) click.echo('Outfile: {}'.format(outfile)) for setting_key in ( 'GenerateCosmicRayMuonsSettings', 'MuonGeometryFilterSettings', 'MuonLossProfileFilterSettings', ): if cfg[setting_key]: click.echo('{}:'.format(setting_key)) for setting, value in cfg[setting_key].items(): click.echo('\t{}: {}'.format(setting, value)) # crate random services if 'random_service_use_gslrng' not in cfg: cfg['random_service_use_gslrng'] = False random_services, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=2, use_gslrng=cfg['random_service_use_gslrng']) # -------------------------------------- # Build IceTray # -------------------------------------- tray = I3Tray() # add random generator to tray context tray.context['I3RandomService'] = random_services[0] tray.AddModule('I3InfiniteSource', 'source', Prefix=cfg['gcd'], Stream=icetray.I3Frame.DAQ) if 'oversampling_factor' not in cfg: cfg['oversampling_factor'] = None if 'oversample_after_proposal' in cfg and \ cfg['oversample_after_proposal']: oversampling_factor_injection = None oversampling_factor_photon = cfg['oversampling_factor'] else: oversampling_factor_injection = cfg['oversampling_factor'] oversampling_factor_photon = None tray.AddSegment(segments.GenerateCosmicRayMuons, 'GenerateCosmicRayMuons', num_events=cfg['n_events_per_run'], **cfg['GenerateCosmicRayMuonsSettings']) # add filter to bias simulation based on track geometry tray.AddModule(MuonGeometryFilter, 'MuonGeometryFilter', **cfg['MuonGeometryFilterSettings']) # add corridor MuonGun bias module: # Events can be biased while keeping ability to properly weight events. # This is not the case for the filter module above. bias_mesc_hotspot_muons(tray, cfg) bias_corridor_muons(tray, cfg) tray.AddModule(DAQFrameMultiplier, 'PreDAQFrameMultiplier', oversampling_factor=oversampling_factor_injection, mctree_keys=['I3MCTree_preMuonProp']) # propagate muons if config exists in config # Note: Snowstorm may perform muon propagation internally if 'muon_propagation_config' in cfg: tray.AddSegment(segments.PropagateMuons, 'propagate_muons', RandomService=random_services[1], **cfg['muon_propagation_config']) else: # In this case we are not propagating the I3MCTree yet, but # are letting this be done by snowstorm propagation # We need to add a key named 'I3MCTree', since snowstorm expects this # It will propagate the particles for us. tray.AddModule('DummyMCTreeRenaming', 'DummyMCTreeRenaming') # add filter to bias simulation based on muon loss profile tray.AddModule(MuonLossProfileFilter, 'MuonLossProfileFilter', **cfg['MuonLossProfileFilterSettings']) # add MuonGun bias module: # Events can be biased while keeping ability to properly weight events. # This is not the case for the filter modules above. bias_muongun_events(tray, cfg) tray.AddModule(DAQFrameMultiplier, 'PostDAQFrameMultiplier', oversampling_factor=oversampling_factor_photon, mctree_keys=['I3MCTree']) # -------------------------------------- # Distance Splits # -------------------------------------- if cfg['distance_splits'] is not None: click.echo('SplittingDistance: {}'.format(cfg['distance_splits'])) distance_splits = np.atleast_1d(cfg['distance_splits']) dom_limits = np.atleast_1d(cfg['threshold_doms']) if len(dom_limits) == 1: dom_limits = np.ones_like(distance_splits) * cfg['threshold_doms'] oversize_factors = np.atleast_1d(cfg['oversize_factors']) order = np.argsort(distance_splits) distance_splits = distance_splits[order] dom_limits = dom_limits[order] oversize_factors = oversize_factors[order] stream_objects = generate_stream_object(distance_splits, dom_limits, oversize_factors) tray.AddModule(OversizeSplitterNSplits, "OversizeSplitterNSplits", thresholds=distance_splits, thresholds_doms=dom_limits, oversize_factors=oversize_factors) for stream_i in stream_objects: outfile_i = stream_i.transform_filepath(outfile) tray.AddModule("I3Writer", "writer_{}".format(stream_i.stream_name), Filename=outfile_i, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ], If=stream_i) click.echo('Output ({}): {}'.format(stream_i.stream_name, outfile_i)) else: click.echo('Output: {}'.format(outfile)) tray.AddModule("I3Writer", "writer", Filename=outfile, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ]) # -------------------------------------- click.echo('Scratch: {}'.format(scratch)) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: cfg = yaml.full_load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace(' ', '0') infile = cfg['infile_pattern'].format(**cfg) infile = infile.replace(' ', '0') click.echo('Run: {}'.format(run_number)) click.echo('Outfile: {}'.format(outfile)) # ----------------------------- # Set PPC environment variables # ----------------------------- ppc_config = cfg['ppc_config'] # define default Environment variables ice_base = '$I3_BUILD/ice-models/resources/models/' default_ppc_environment_variables = { 'PPCTABLESDIR': ice_base + 'spice_bfr-dv1_complete', 'OGPU': '1', # makes sure only GPUs are used (with OpenCL version) } ppc_environment_variables = dict(default_ppc_environment_variables) ppc_environment_variables.update(ppc_config['environment_variables']) # define default PPC arguments default_ppc_arguments = { 'MCTree': 'I3MCTree', } if 'CUDA_VISIBLE_DEVICES' in os.environ: default_ppc_arguments['gpu'] = int(os.environ['CUDA_VISIBLE_DEVICES']) ppc_arguments = dict(default_ppc_arguments) ppc_arguments.update(ppc_config['arguments']) click.echo('PPC Settings:') for key, value in ppc_environment_variables.items(): click.echo('\t{}: {}'.format(key, os.path.expandvars(value))) os.putenv(key, os.path.expandvars(value)) click.echo('PPC Arguments:') for key, value in ppc_arguments.items(): click.echo('\t{}: {}'.format(key, value)) # importing ppc must be done *after* setting the environment variables from icecube import ppc # ------------------------------ # get random service random_services, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=1, use_gslrng=cfg['random_service_use_gslrng']) random_service = random_services[0] # -------------------------------------- # Build IceTray # -------------------------------------- tray = I3Tray() tray.AddModule('I3Reader', 'i3 reader', FilenameList=[cfg['gcd_pass2'], infile]) # run PPC tray.context["I3RandomService"] = random_service tray.AddModule("i3ppc", 'ppc', **ppc_arguments) # rename MCPESeriesMap to I3MCPESeriesMap tray.Add("Rename", keys=["MCPESeriesMap", "I3MCPESeriesMap"]) click.echo('Output: {}'.format(outfile)) tray.AddModule("I3Writer", "writer", Filename=outfile, Streams=[ icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation, icetray.I3Frame.Stream('M'), icetray.I3Frame.Stream('S'), icetray.I3Frame.DAQ, icetray.I3Frame.Physics ]) # -------------------------------------- click.echo('Scratch: {}'.format(scratch)) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def process_single_stream(cfg, infile, outfile): click.echo('Input: {}'.format(infile)) hybrid_mode = (cfg['clsim_hybrid_mode'] and cfg['icemodel'].lower() != 'spicelea') ignore_muon_light = (cfg['clsim_ignore_muon_light'] and cfg['clsim_hybrid_mode']) click.echo('UseGPUs: {}'.format(cfg['clsim_usegpus'])) click.echo('IceModel: {}'.format(cfg['icemodel'])) if not cfg['icemodel_location'] is None: click.echo('IceModelLocation: {}'.format(cfg['icemodel_location'])) click.echo('DomOversize {}'.format(cfg['clsim_dom_oversize'])) click.echo('UnshadowedFraction: {0:.2f}'.format( cfg['clsim_unshadowed_fraction'])) click.echo('HybridMode: {}'.format(hybrid_mode)) click.echo('IgnoreMuonLight: {}'.format(ignore_muon_light)) click.echo('KeepMCPE: {}'.format(cfg['clsim_keep_mcpe'])) click.echo('Output: {}'.format(outfile)) tray = I3Tray() tray.context['I3FileStager'] = dataio.get_stagers() random_services, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=process_single_stream.n_streams) random_service = random_services[process_single_stream.i_th_stream] tray.context['I3RandomService'] = random_service tray.Add('I3Reader', FilenameList=[cfg['gcd'], infile]) if hybrid_mode: cascade_tables = segments.LoadCascadeTables(IceModel=cfg['icemodel'], TablePath=SPLINE_TABLES) else: cascade_tables = None if cfg['clsim_usegpus']: use_gpus = True use_cpus = False else: use_gpus = True use_cpus = False tray.AddSegment(segments.PropagatePhotons, "PropagatePhotons", RandomService=random_service, MaxParallelEvents=MAX_PARALLEL_EVENTS, KeepIndividualMaps=cfg['clsim_keep_mcpe'], IceModel=cfg['icemodel'], IceModelLocation=cfg['icemodel_location'], UnshadowedFraction=cfg['clsim_unshadowed_fraction'], IgnoreMuons=ignore_muon_light, HybridMode=hybrid_mode, UseGPUs=use_gpus, UseAllCPUCores=use_cpus, DOMOversizeFactor=cfg['clsim_dom_oversize'], CascadeService=cascade_tables) outfile = outfile.replace(' ', '0') tray.AddModule("I3Writer", "writer", Filename=outfile, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ]) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: cfg = yaml.load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) if cfg['distance_splits'] is not None: click.echo('SplittingDistances: {}'.format(cfg['distance_splits'])) click.echo('Oversizefactors: {}'.format(cfg['oversize_factors'])) click.echo('NEvents: {}'.format(cfg['n_events_per_run'])) click.echo('EMin: {}'.format(cfg['e_min'])) click.echo('EMax: {}'.format(cfg['e_max'])) click.echo('EBreak: {}'.format(cfg['muongun_e_break'])) click.echo('Gamma: {}'.format(cfg['gamma'])) click.echo('ZenithMin: {}'.format(cfg['zenith_min'])) click.echo('ZenithMax: {}'.format(cfg['zenith_max'])) tray = I3Tray() random_service, random_service_prop, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed']) tray.context['I3RandomService'] = random_service tray.AddModule("I3InfiniteSource", "TheSource", Prefix=cfg['gcd'], Stream=icetray.I3Frame.DAQ) tray.AddSegment(segments.GenerateSingleMuons, "GenerateCosmicRayMuons", NumEvents=cfg['n_events_per_run'], FromEnergy=cfg['e_min'] * icetray.I3Units.GeV, ToEnergy=cfg['e_max'] * icetray.I3Units.GeV, BreakEnergy=cfg['muongun_e_break'] * icetray.I3Units.GeV, GammaIndex=cfg['gamma'], ZenithRange=[ cfg['zenith_min'] * icetray.I3Units.deg, cfg['zenith_max'] * icetray.I3Units.deg ]) tray.AddSegment(segments.PropagateMuons, "PropagateMuons", RandomService=random_service_prop) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace(' ', '0') if cfg['distance_splits'] is not None: click.echo('SplittingDistance: {}'.format(cfg['distance_splits'])) distance_splits = np.atleast_1d(cfg['distance_splits']) dom_limits = np.atleast_1d(cfg['threshold_doms']) if len(dom_limits) == 1: dom_limits = np.ones_like(distance_splits) * cfg['threshold_doms'] oversize_factors = np.atleast_1d(cfg['oversize_factors']) order = np.argsort(distance_splits) distance_splits = distance_splits[order] dom_limits = dom_limits[order] oversize_factors = oversize_factors[order] stream_objects = generate_stream_object(distance_splits, dom_limits, oversize_factors) tray.AddModule(OversizeSplitterNSplits, "OversizeSplitterNSplits", thresholds=distance_splits, thresholds_doms=dom_limits, oversize_factors=oversize_factors) for stream_i in stream_objects: outfile_i = stream_i.transform_filepath(outfile) tray.AddModule("I3Writer", "writer_{}".format(stream_i.stream_name), Filename=outfile_i, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ], If=stream_i) click.echo('Output ({}): {}'.format(stream_i.stream_name, outfile_i)) else: click.echo('Output: {}'.format(outfile)) tray.AddModule("I3Writer", "writer", Filename=outfile, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ]) click.echo('Scratch: {}'.format(scratch)) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: cfg = yaml.full_load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace(' ', '0') # ------------------------------ # get list of files for this run # ------------------------------ import_cfg = cfg['event_import_settings'] glob_files = import_cfg['input_file_glob_list'] if isinstance(glob_files, str): # single string provided files = glob.glob(glob_files.format(run_number=run_number)) else: # list of file globs provided files = [] for file_pattern in glob_files: files.extend(glob.glob(file_pattern.format(run_number=run_number))) # sort files files = sorted(files) # ------------------------------ click.echo('Run: {}'.format(run_number)) click.echo('Outfile: {}'.format(outfile)) click.echo('Keys to import: {}'.format(import_cfg['keys_to_import'])) click.echo('input Files:') for file in files: click.echo('\t{}'.format(file)) # crate random services if 'random_service_use_gslrng' not in cfg: cfg['random_service_use_gslrng'] = False random_services, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=2, use_gslrng=cfg['random_service_use_gslrng']) # -------------------------------------- # Build IceTray # -------------------------------------- tray = I3Tray() # import events from another I3-file if 'num_events' not in import_cfg: import_cfg['num_events'] = None tray.AddModule( ImportEvents, 'ImportEvents', files=files, num_events=import_cfg['num_events'], keys_to_import=import_cfg['keys_to_import'], rename_dict=import_cfg['rename_dict'], mctree_name=import_cfg['mctree_name'], ) # propagate muons if config exists in config # Note: Snowstorm may perform muon propagation internally if 'muon_propagation_config' in cfg: tray.AddSegment(segments.PropagateMuons, 'propagate_muons', RandomService=random_services[1], **cfg['muon_propagation_config']) tray.AddModule(DAQFrameMultiplier, 'DAQFrameMultiplier', oversampling_factor=cfg['oversampling_factor'], mctree_keys=[import_cfg['mctree_name']]) # -------------------------------------- # Distance Splits # -------------------------------------- if cfg['distance_splits'] is not None: click.echo('SplittingDistance: {}'.format(cfg['distance_splits'])) distance_splits = np.atleast_1d(cfg['distance_splits']) dom_limits = np.atleast_1d(cfg['threshold_doms']) if len(dom_limits) == 1: dom_limits = np.ones_like(distance_splits) * cfg['threshold_doms'] oversize_factors = np.atleast_1d(cfg['oversize_factors']) order = np.argsort(distance_splits) distance_splits = distance_splits[order] dom_limits = dom_limits[order] oversize_factors = oversize_factors[order] stream_objects = generate_stream_object(distance_splits, dom_limits, oversize_factors) tray.AddModule(OversizeSplitterNSplits, "OversizeSplitterNSplits", thresholds=distance_splits, thresholds_doms=dom_limits, oversize_factors=oversize_factors) for stream_i in stream_objects: outfile_i = stream_i.transform_filepath(outfile) tray.AddModule("I3Writer", "writer_{}".format(stream_i.stream_name), Filename=outfile_i, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ], If=stream_i) click.echo('Output ({}): {}'.format(stream_i.stream_name, outfile_i)) else: click.echo('Output: {}'.format(outfile)) tray.AddModule("I3Writer", "writer", Filename=outfile, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ]) # -------------------------------------- click.echo('Scratch: {}'.format(scratch)) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: cfg = yaml.full_load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace(' ', '0') click.echo('Run: {}'.format(run_number)) click.echo('Outfile: {}'.format(outfile)) click.echo('Azimuth: [{},{}]'.format(*cfg['azimuth_range'])) click.echo('Zenith: [{},{}]'.format(*cfg['zenith_range'])) click.echo('Energy: [{},{}]'.format(*cfg['primary_energy_range'])) click.echo('Vertex x: [{},{}]'.format(*cfg['x_range'])) click.echo('Vertex y: [{},{}]'.format(*cfg['y_range'])) click.echo('Vertex z: [{},{}]'.format(*cfg['z_range'])) # crate random services if 'random_service_use_gslrng' not in cfg: cfg['random_service_use_gslrng'] = False random_services, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=2, use_gslrng=cfg['random_service_use_gslrng']) # -------------------------------------- # Build IceTray # -------------------------------------- tray = I3Tray() tray.AddModule( 'I3InfiniteSource', 'source', # Prefix=gcdfile, Stream=icetray.I3Frame.DAQ) if 'max_vertex_distance' not in cfg: cfg['max_vertex_distance'] = None if 'constant_vars' not in cfg: cfg['constant_vars'] = None if 'sample_uniformly_on_sphere' not in cfg: cfg['sample_uniformly_on_sphere'] = False if 'oversample_after_proposal' in cfg and \ cfg['oversample_after_proposal']: oversampling_factor_injection = None oversampling_factor_photon = cfg['oversampling_factor'] else: oversampling_factor_injection = cfg['oversampling_factor'] oversampling_factor_photon = None tray.AddModule( CascadeFactory, 'make_cascades', azimuth_range=cfg['azimuth_range'], zenith_range=cfg['zenith_range'], sample_uniformly_on_sphere=cfg['sample_uniformly_on_sphere'], primary_energy_range=cfg['primary_energy_range'], fractional_energy_in_hadrons_range=cfg[ 'fractional_energy_in_hadrons_range'], time_range=cfg['time_range'], x_range=cfg['x_range'], y_range=cfg['y_range'], z_range=cfg['z_range'], max_vertex_distance=cfg['max_vertex_distance'], flavors=cfg['flavors'], interaction_types=cfg['interaction_types'], num_events=cfg['n_events_per_run'], oversampling_factor=oversampling_factor_injection, random_state=cfg['seed'], random_service=random_services[0], constant_vars=cfg['constant_vars'], ) # propagate muons if config exists in config # Note: Snowstorm may perform muon propagation internally if 'muon_propagation_config' in cfg: tray.AddSegment(segments.PropagateMuons, 'propagate_muons', RandomService=random_services[1], **cfg['muon_propagation_config']) else: # In this case we are not propagating the I3MCTree yet, but # are letting this be done by snowstorm propagation # We need to add a key named 'I3MCTree', since snowstorm expects this # It will propagate the particles for us. tray.AddModule(DummyMCTreeRenaming, 'DummyMCTreeRenaming') tray.AddModule(DAQFrameMultiplier, 'DAQFrameMultiplier', oversampling_factor=oversampling_factor_photon) # -------------------------------------- # Distance Splits # -------------------------------------- if cfg['distance_splits'] is not None: click.echo('SplittingDistance: {}'.format(cfg['distance_splits'])) distance_splits = np.atleast_1d(cfg['distance_splits']) dom_limits = np.atleast_1d(cfg['threshold_doms']) if len(dom_limits) == 1: dom_limits = np.ones_like(distance_splits) * cfg['threshold_doms'] oversize_factors = np.atleast_1d(cfg['oversize_factors']) order = np.argsort(distance_splits) distance_splits = distance_splits[order] dom_limits = dom_limits[order] oversize_factors = oversize_factors[order] stream_objects = generate_stream_object(distance_splits, dom_limits, oversize_factors) tray.AddModule(OversizeSplitterNSplits, "OversizeSplitterNSplits", thresholds=distance_splits, thresholds_doms=dom_limits, oversize_factors=oversize_factors) for stream_i in stream_objects: outfile_i = stream_i.transform_filepath(outfile) tray.AddModule("I3Writer", "writer_{}".format(stream_i.stream_name), Filename=outfile_i, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ], If=stream_i) click.echo('Output ({}): {}'.format(stream_i.stream_name, outfile_i)) else: click.echo('Output: {}'.format(outfile)) tray.AddModule("I3Writer", "writer", Filename=outfile, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ]) # -------------------------------------- click.echo('Scratch: {}'.format(scratch)) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: cfg = yaml.full_load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace(' ', '0') # ------------------------------ # get list of files for this run # ------------------------------ import_cfg = cfg['event_import_settings'] glob_files = import_cfg['input_file_glob_list'] import_cfg['run_number'] = cfg['run_number'] import_cfg['dataset_number'] = cfg['dataset_number'] import_cfg['folder_num_pre_offset'] = cfg['run_number']//1000 import_cfg['folder_num'] = ( import_cfg['folder_offset'] + cfg['run_number']//1000 ) import_cfg['folder_pattern'] = import_cfg['folder_pattern'].format( **import_cfg) import_cfg['run_folder'] = import_cfg['folder_pattern'].format( **import_cfg) if isinstance(glob_files, str): # single string provided files = glob.glob(glob_files.format(**import_cfg)) else: # list of file globs provided files = [] for file_pattern in glob_files: files.extend(glob.glob(file_pattern.format(**import_cfg))) # sort files files = sorted(files) # ------------------------------ click.echo('Run: {}'.format(run_number)) click.echo('Outfile: {}'.format(outfile)) click.echo('Keys to import: {}'.format(import_cfg['keys_to_import'])) click.echo('input Files:') for file in files: click.echo('\t{}'.format(file)) # crate random services if 'random_service_use_gslrng' not in cfg: cfg['random_service_use_gslrng'] = False random_services, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=3, use_gslrng=cfg['random_service_use_gslrng']) # -------------------------------------- # Build IceTray # -------------------------------------- tray = I3Tray() # import events from another I3-file if 'num_events' not in import_cfg: import_cfg['num_events'] = None tray.AddModule( ImportEvents, 'ImportEvents', files=files, num_events=import_cfg['num_events'], keys_to_import=import_cfg['keys_to_import'], rename_dict=import_cfg['rename_dict'], mctree_name=import_cfg['mctree_name'], ) # inject coincident muon from some direction as neutrino if 'mctree_name' not in cfg['veto_muon_injection_config']: cfg['veto_muon_injection_config']['mctree_name'] = ( import_cfg['mctree_name'] ) tray.AddModule( InjectSingleVetoMuon, 'InjectSingleVetoMuon', random_service=random_services[0], **cfg['veto_muon_injection_config'] ) t_name = import_cfg['mctree_name'] # # rename I3MCTrees so that we can run PROPOSAL # def rename_keys(frame, rename_dict): # for key, new_name in rename_dict.items(): # if key in frame: # frame[new_name] = frame[key] # del frame[key] # print('key', key) # rename_dict = { # t_name: '_' + t_name, # t_name + '_preMuonProp': '_' + t_name + '_preMuonProp', # t_name + '_preMuonProp_RNGState': ( # '_' + t_name + '_preMuonProp_RNGState', # ), # t_name + '_veto_muon': 'I3MCTree_preMuonProp', # } # tray.AddModule(rename_keys, 'TempRename', rename_dict=rename_dict) # temporarily save MMCTrackList tray.AddModule('Rename', keys=['MMCTrackList', '_MMCTrackList']) # propagate injected muon cfg['muon_propagation_config'].update({ 'InputMCTreeName': t_name + 'VetoMuon_preMuonProp', 'OutputMCTreeName': t_name + 'VetoMuon', }) tray.AddSegment(segments.PropagateMuons, 'propagate_muons', RandomService=random_services[1], **cfg['muon_propagation_config']) # undo renaming tray.AddModule('Rename', keys=['MMCTrackList', 'MMCTrackListVetoMuon']) tray.AddModule('Rename', keys=['_MMCTrackList', 'MMCTrackList']) # # now revert renaming # rename_dict = { # 'I3MCTree': t_name + '_veto_muon', # 'I3MCTree_preMuonProp': t_name + '_preMuonProp_veto_muon', # '_' + t_name: t_name, # '_' + t_name + '_preMuonProp': t_name + '_preMuonProp', # '_' + t_name + '_preMuonProp_RNGState': ( # t_name + '_preMuonProp_RNGState', # ), # } # tray.AddModule(rename_keys, 'UndoRenaming', rename_dict=rename_dict) # create combined I3MCTree for CLSIM tray.AddModule( CombineMCTrees, 'CombineMCTrees', tree1=t_name, tree2=t_name + 'VetoMuon', output_key='CombinedMuonVetoI3MCTree', ) # Bias simulation if desired if 'ApplyBaseSimulationBias' in cfg and cfg['ApplyBaseSimulationBias']: tray.AddModule( BaseSimulationBias, 'BaseSimulationBias', random_service=random_services[2], **cfg['BaseSimulationBiasSettings'] ) # -------------------------------------- # Distance Splits # -------------------------------------- if cfg['distance_splits'] is not None: click.echo('SplittingDistance: {}'.format( cfg['distance_splits'])) distance_splits = np.atleast_1d(cfg['distance_splits']) dom_limits = np.atleast_1d(cfg['threshold_doms']) if len(dom_limits) == 1: dom_limits = np.ones_like(distance_splits) * cfg['threshold_doms'] oversize_factors = np.atleast_1d(cfg['oversize_factors']) order = np.argsort(distance_splits) distance_splits = distance_splits[order] dom_limits = dom_limits[order] oversize_factors = oversize_factors[order] stream_objects = generate_stream_object(distance_splits, dom_limits, oversize_factors) tray.AddModule(OversizeSplitterNSplits, "OversizeSplitterNSplits", thresholds=distance_splits, thresholds_doms=dom_limits, oversize_factors=oversize_factors) for stream_i in stream_objects: outfile_i = stream_i.transform_filepath(outfile) tray.AddModule("I3Writer", "writer_{}".format(stream_i.stream_name), Filename=outfile_i, Streams=[icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M')], If=stream_i) click.echo('Output ({}): {}'.format(stream_i.stream_name, outfile_i)) else: click.echo('Output: {}'.format(outfile)) tray.AddModule("I3Writer", "writer", Filename=outfile, Streams=[icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M')]) # -------------------------------------- click.echo('Scratch: {}'.format(scratch)) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: cfg = yaml.load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace(' ', '0') click.echo('Run: {}'.format(run_number)) click.echo('Outfile: {}'.format(outfile)) click.echo('Azimuth: [{},{}]'.format(cfg['azimuth_min'], cfg['azimuth_max'])) click.echo('Zenith: [{},{}]'.format(cfg['zenith_min'], cfg['zenith_max'])) click.echo('Energy: [{},{}]'.format(cfg['e_min'], cfg['e_max'])) # crate random services random_services, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=2) # create convex hull if 'use_convex_hull' in cfg and cfg['use_convex_hull']: # hardcode icecube corner points # ToDo: read from geometry file points = [ [-570.90002441, -125.13999939, 501], # string 31 [-256.14001465, -521.08001709, 501], # string 1 [ 361. , -422.82998657, 501], # string 6 [ 576.36999512, 170.91999817, 501], # string 50 [ 338.44000244, 463.72000122, 501], # string 74 [ 101.04000092, 412.79000854, 501], # string 72 [ 22.11000061, 509.5 , 501], # string 78 [-347.88000488, 451.51998901, 501], # string 75 [-570.90002441, -125.13999939, -502], # string 31 [-256.14001465, -521.08001709, -502], # string 1 [ 361. , -422.82998657, -502], # string 6 [ 576.36999512, 170.91999817, -502], # string 50 [ 338.44000244, 463.72000122, -502], # string 74 [ 101.04000092, 412.79000854, -502], # string 72 [ 22.11000061, 509.5 , -502], # string 78 [-347.88000488, 451.51998901, -502], # string 75 ] convex_hull = ConvexHull(points) else: convex_hull = None if not 'extend_past_hull' in cfg: cfg['extend_past_hull'] = 0.0 # create muon muon = create_muon( azimuth_range=[cfg['azimuth_min'],cfg['azimuth_max']], zenith_range=[cfg['zenith_min'],cfg['zenith_max']], energy_range=[cfg['e_min'],cfg['e_max']], anchor_time_range=cfg['anchor_time_range'], anchor_x_range=cfg['anchor_x_range'], anchor_y_range=cfg['anchor_y_range'], anchor_z_range=cfg['anchor_z_range'], length_to_go_back=cfg['length_to_go_back'], convex_hull=convex_hull, extend_past_hull=cfg['extend_past_hull'], random_service=random_services[0], ) #-------------------------------------- # Build IceTray #-------------------------------------- tray = I3Tray() tray.AddModule('I3InfiniteSource', 'source', # Prefix=gcdfile, Stream=icetray.I3Frame.DAQ) tray.AddModule(ParticleMultiplier, 'make_particles', num_events=cfg['n_events_per_run'], primary= muon) tray.AddSegment(segments.PropagateMuons, 'propagate_muons', RandomService=random_services[1], **cfg['muon_propagation_config']) #-------------------------------------- # Distance Splits #-------------------------------------- if cfg['distance_splits'] is not None: click.echo('SplittingDistance: {}'.format( cfg['distance_splits'])) distance_splits = np.atleast_1d(cfg['distance_splits']) dom_limits = np.atleast_1d(cfg['threshold_doms']) if len(dom_limits) == 1: dom_limits = np.ones_like(distance_splits) * cfg['threshold_doms'] oversize_factors = np.atleast_1d(cfg['oversize_factors']) order = np.argsort(distance_splits) distance_splits = distance_splits[order] dom_limits = dom_limits[order] oversize_factors = oversize_factors[order] stream_objects = generate_stream_object(distance_splits, dom_limits, oversize_factors) tray.AddModule(OversizeSplitterNSplits, "OversizeSplitterNSplits", thresholds=distance_splits, thresholds_doms=dom_limits, oversize_factors=oversize_factors) for stream_i in stream_objects: outfile_i = stream_i.transform_filepath(outfile) tray.AddModule("I3Writer", "writer_{}".format(stream_i.stream_name), Filename=outfile_i, Streams=[icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M')], If=stream_i) click.echo('Output ({}): {}'.format(stream_i.stream_name, outfile_i)) else: click.echo('Output: {}'.format(outfile)) tray.AddModule("I3Writer", "writer", Filename=outfile, Streams=[icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M')]) #-------------------------------------- click.echo('Scratch: {}'.format(scratch)) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def process_single_stream(cfg, infile, outfile): click.echo('Input: {}'.format(infile)) hybrid_mode = (cfg['clsim_hybrid_mode'] and cfg['icemodel'].lower() != 'spicelea') ignore_muon_light = (cfg['clsim_ignore_muon_light'] and cfg['clsim_hybrid_mode']) click.echo('UseGPUs: {}'.format(cfg['clsim_usegpus'])) click.echo('IceModel: {}'.format(cfg['icemodel'])) if not cfg['icemodel_location'] is None: click.echo('IceModelLocation: {}'.format(cfg['icemodel_location'])) click.echo('DomOversize {}'.format(cfg['clsim_dom_oversize'])) click.echo('UnshadowedFraction: {0:.2f}'.format( cfg['clsim_unshadowed_fraction'])) click.echo('HybridMode: {}'.format(hybrid_mode)) click.echo('IgnoreMuonLight: {}'.format(ignore_muon_light)) click.echo('KeepMCPE: {}'.format(cfg['clsim_keep_mcpe'])) click.echo('Output: {}'.format(outfile)) tray = I3Tray() tray.context['I3FileStager'] = dataio.get_stagers() random_services, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=process_single_stream.n_streams) random_service = random_services[process_single_stream.i_th_stream] tray.context['I3RandomService'] = random_service tray.Add('I3Reader', FilenameList=[cfg['gcd'], infile]) if hybrid_mode: cascade_tables = segments.LoadCascadeTables( IceModel=cfg['icemodel'], TablePath=cfg['spline_table_dir']) else: cascade_tables = None if cfg['clsim_usegpus']: use_gpus = True use_cpus = False else: use_gpus = True use_cpus = False if 'additional_clsim_params' in cfg: additional_clsim_params = cfg['additional_clsim_params'] else: additional_clsim_params = {} if not cfg['clsim_input_is_sliced']: MCTreeName = "I3MCTree" MMCTrackListName = "MMCTrackList" else: MCTreeName = "I3MCTree_sliced" MMCTrackListName = None #use_gpus=False #use_cpus=True tray.AddSegment(segments.PropagatePhotons, "PropagatePhotons", GCDFile=cfg['gcd'], RandomService=random_service, KeepIndividualMaps=cfg['clsim_keep_mcpe'], IceModel=cfg['icemodel'], IceModelLocation=cfg['icemodel_location'], UnshadowedFraction=cfg['clsim_unshadowed_fraction'], IgnoreMuons=ignore_muon_light, HybridMode=hybrid_mode, UseGPUs=use_gpus, UseAllCPUCores=use_cpus, DOMOversizeFactor=cfg['clsim_dom_oversize'], CascadeService=cascade_tables, **additional_clsim_params) # tray.AddSegment(clsim.I3CLSimMakeHits, "makeCLSimHits", # GCDFile = cfg['gcd'], # PhotonSeriesName = cfg['photonSeriesName'], # MCTreeName = MCTreeName, # MMCTrackListName = MMCTrackListName, # RandomService = random_service, # MCPESeriesName = cfg['mcpe_series_map'], # UnshadowedFraction = cfg['clsim_unshadowed_fraction'], # UseGPUs = use_gpus, # UseCPUs = use_cpus, # IceModelLocation = os.path.expandvars("$I3_BUILD/ice-models/resources/models/spice_lea"), # ) outfile = outfile.replace(' ', '0') tray.AddModule("I3Writer", "writer", Filename=outfile, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ]) tray.Execute() del tray
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: if int(yaml.__version__[0]) < 5: # backwards compatibility for yaml versions before version 5 cfg = yaml.load(stream) else: cfg = yaml.full_load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) infile = cfg['infile_pattern'].format(**cfg) infile = infile.replace(' ', '0') infile = infile.replace('Level0.{}'.format(cfg['previous_step']), 'Level0.{}'.format(cfg['previous_step'] % 10)) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace('Level0.{}'.format(cfg['step']), 'Level0.{}'.format(cfg['step'] % 10)) outfile = outfile.replace(' ', '0') outfile = outfile.replace('2012_pass2', 'pass2') print('Outfile != $FINAL_OUT clean up for crashed scripts not possible!') tray = I3Tray() tray.context['I3FileStager'] = dataio.get_stagers() random_services, run_id = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=1) random_service = random_services[0] tray.context['I3RandomService'] = random_service tray.Add('I3Reader', FilenameList=[cfg['gcd_pass2'], infile]) if run_number < cfg['det_pass2_keep_all_upto']: cfg['det_keep_mc_hits'] = True cfg['det_keep_propagated_mc_tree'] = True cfg['det_keep_mc_pulses'] = True tray.AddSegment(segments.DetectorSim, "Detector5Sim", RandomService='I3RandomService', RunID=run_id, GCDFile=cfg['gcd_pass2'], KeepMCHits=cfg['det_keep_mc_hits'], KeepPropagatedMCTree=cfg['det_keep_propagated_mc_tree'], KeepMCPulses=cfg['det_keep_mc_pulses'], SkipNoiseGenerator=cfg['det_skip_noise_generation'], LowMem=cfg['det_low_mem'], InputPESeriesMapName=MCPE_SERIES_MAP, BeaconLaunches=cfg['det_add_beacon_launches'], FilterTrigger=cfg['det_filter_trigger']) tray.AddModule("I3Writer", "EventWriter", filename=outfile, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation ]) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: cfg = yaml.load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) infile = cfg['infile_pattern'].format(**cfg) infile = infile.replace(' ', '0') infile = infile.replace('Level0.{}'.format(cfg['previous_step']), 'Level0.{}'.format(cfg['previous_step'] % 10)) infile = infile.replace('2012_pass2', 'pass2') if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace('Level0.{}'.format(cfg['step']), 'Level0.{}'.format(cfg['step'] % 10)) outfile = outfile.replace(' ', '0') outfile = outfile.replace('2012_pass2', 'pass2') print('Outfile != $FINAL_OUT clean up for crashed scripts not possible!') tray = I3Tray() """The main L1 script""" tray.AddModule('I3Reader', 'i3 reader', FilenameList=[cfg['gcd_pass2'], infile]) # run online filters online_kwargs = {} if SPLINE_TABLES: online_kwargs.update({ 'SplineRecoAmplitudeTable': os.path.join(SPLINE_TABLES, 'InfBareMu_mie_abs_z20a10.fits'), 'SplineRecoTimingTable': os.path.join(SPLINE_TABLES, 'InfBareMu_mie_prob_z20a10.fits'), 'hese_followup_base_GCD_filename': cfg['gcd_pass2'], }) if cfg['L1_pass2_run_gfu'] is not None: online_kwargs['gfu_enabled'] = cfg['L1_pass2_run_gfu'] tray.AddSegment(OnlineFilter, "OnlineFilter", decode=False, simulation=True, vemcal_enabled=False, **online_kwargs) # make random service _, seed = create_random_services(dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=0) filter_mask_randoms = phys_services.I3GSLRandomService(seed) # override MinBias Prescale filterconfigs = filter_globals.filter_pairs + filter_globals.sdst_pairs print(cfg['L1_min_bias_prescale']) if cfg['L1_min_bias_prescale']: for i, filtertuple in enumerate(filterconfigs): if filtertuple[0] == filter_globals.FilterMinBias: del filterconfigs[i] filterconfigs.append( (filtertuple[0], cfg['L1_min_bias_prescale'])) break print(filterconfigs) # Generate filter Masks for all P frames tray.AddModule(filter_tools.FilterMaskMaker, "MakeFilterMasks", OutputMaskName=filter_globals.filter_mask, FilterConfigs=filterconfigs, RandomService=filter_mask_randoms) # Merge the FilterMasks tray.AddModule("OrPframeFilterMasks", "make_q_filtermask", InputName=filter_globals.filter_mask, OutputName=filter_globals.qfilter_mask) # Q+P frame specific keep module needs to go first, as KeepFromSubstram # will rename things, let's rename post keep. def is_Q(frame): return frame.Stop == frame.DAQ simulation_keeps = [ 'BackgroundI3MCTree', 'BackgroundI3MCTreePEcounts', 'BackgroundI3MCPESeriesMap', 'BackgroundI3MCTree_preMuonProp', 'BackgroundMMCTrackList', 'BeaconLaunches', 'CorsikaInteractionHeight', 'CorsikaWeightMap', 'EventProperties', 'GenerationSpec', 'I3LinearizedMCTree', 'I3MCTree', 'I3MCTreePEcounts', 'I3MCTree_preMuonProp', 'I3MCPESeriesMap', 'I3MCPulseSeriesMap', 'I3MCPulseSeriesMapParticleIDMap', 'I3MCWeightDict', 'LeptonInjectorProperties', 'MCHitSeriesMap', 'MCPrimary', 'MCPrimaryInfo', 'MMCTrackList', 'PolyplopiaInfo', 'PolyplopiaPrimary', 'RNGState', 'SignalI3MCPEs', 'SimTrimmer', 'TimeShift', 'WIMP_params' ] + muongun_keys keep_before_merge = filter_globals.q_frame_keeps + [ 'InIceDSTPulses', 'IceTopDSTPulses', 'CalibratedWaveformRange', 'UncleanedInIcePulsesTimeRange', 'SplitUncleanedInIcePulses', 'SplitUncleanedInIcePulsesTimeRange', 'SplitUncleanedInIceDSTPulsesTimeRange', 'CalibrationErrata', 'SaturationWindows', 'InIceRawData', 'IceTopRawData' ] + simulation_keeps tray.AddModule("Keep", "keep_before_merge", keys=keep_before_merge, If=is_Q) tray.AddModule("I3IcePickModule<FilterMaskFilter>", "filterMaskCheckAll", FilterNameList=filter_globals.filter_streams, FilterResultName=filter_globals.qfilter_mask, DecisionName="PassedAnyFilter", DiscardEvents=False, Streams=[icetray.I3Frame.DAQ]) def do_save_just_superdst(frame): if frame.Has("PassedAnyFilter"): if not frame["PassedAnyFilter"].value: return True else: return False else: print("Failed to find key frame Bool!!") return False keep_only_superdsts = filter_globals.keep_nofilterpass + [ 'PassedAnyFilter', 'InIceDSTPulses', 'IceTopDSTPulses', 'SplitUncleanedInIcePulses', 'SplitUncleanedInIcePulsesTimeRange', 'SplitUncleanedInIceDSTPulsesTimeRange', 'RNGState' ] + simulation_keeps tray.AddModule("Keep", "KeepOnlySuperDSTs", keys=keep_only_superdsts, If=do_save_just_superdst) tray.AddModule("I3IcePickModule<FilterMaskFilter>", "filterMaskCheckSDST", FilterNameList=filter_globals.sdst_streams, FilterResultName=filter_globals.qfilter_mask, DecisionName="PassedKeepSuperDSTOnly", DiscardEvents=False, Streams=[icetray.I3Frame.DAQ]) def dont_save_superdst(frame): if frame.Has("PassedKeepSuperDSTOnly") and \ frame.Has("PassedAnyFilter"): if frame["PassedAnyFilter"].value: return False elif not frame["PassedKeepSuperDSTOnly"].value: return True else: return False else: print("Failed to find key frame Bool!!") return False tray.AddModule("Keep", "KeepOnlyDSTs", keys=filter_globals.keep_dst_only + [ "PassedAnyFilter", "PassedKeepSuperDSTOnly", filter_globals.eventheader ] + muongun_keys, If=dont_save_superdst) tray.AddModule("KeepFromSubstream", "null_stream", StreamName=filter_globals.NullSplitter, KeepKeys=filter_globals.null_split_keeps) in_ice_keeps = filter_globals.inice_split_keeps + \ filter_globals.onlinel2filter_keeps in_ice_keeps = in_ice_keeps + [ 'I3EventHeader', 'SplitUncleanedInIcePulses', 'TriggerSplitterLaunchWindow', 'I3TriggerHierarchy', 'GCFilter_GCFilterMJD' ] + muongun_keys tray.AddModule( "Keep", "inice_keeps", keys=in_ice_keeps, If=which_split(split_name=filter_globals.InIceSplitter), ) tray.AddModule("KeepFromSubstream", "icetop_split_stream", StreamName=filter_globals.IceTopSplitter, KeepKeys=filter_globals.icetop_split_keeps) tray.AddModule("I3IcePickModule<FilterMaskFilter>", "filterMaskCheck", FilterNameList=filter_globals.filters_keeping_allraw, FilterResultName=filter_globals.qfilter_mask, DecisionName="PassedConventional", DiscardEvents=False, Streams=[icetray.I3Frame.DAQ]) def I3RawDataCleaner(frame): if not (('PassedConventional' in frame and frame['PassedConventional'].value == True) or ('SimTrimmer' in frame and frame['SimTrimmer'].value == True)): frame.Delete('InIceRawData') frame.Delete('IceTopRawData') tray.AddModule(I3RawDataCleaner, "CleanErrataForConventional", Streams=[icetray.I3Frame.DAQ]) tray.AddModule("I3Writer", "EventWriter", filename=outfile, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation ]) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: if int(yaml.__version__[0]) < 5: # backwards compatibility for yaml versions before version 5 cfg = yaml.load(stream) else: cfg = yaml.full_load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace(' ', '0') click.echo('NEvents: {}'.format(cfg['n_events_per_run'])) click.echo('EMin: {}'.format(cfg['e_min'])) click.echo('EMax: {}'.format(cfg['e_max'])) click.echo('Gamma: {}'.format(cfg['gamma'])) click.echo('ZenithMin: {}'.format(cfg['zenith_min'])) click.echo('ZenithMax: {}'.format(cfg['zenith_max'])) click.echo('AzimuthMin: {}'.format(cfg['azimuth_min'])) click.echo('AzimuthMax: {}'.format(cfg['azimuth_max'])) if cfg['neutrino_flavor'] is None: click.echo('NeutrinoTypes: {}'.format(cfg['neutrino_types'])) click.echo('PrimaryTypeRatio: {}'.format(cfg['primary_type_ratio'])) else: click.echo('NeutrinoFlavor: {}'.format(cfg['neutrino_flavor'])) if 'ApplyBaseSimulationBias' in cfg and cfg['ApplyBaseSimulationBias']: click.echo('Apply simulation bias: True') else: click.echo('Apply simulation bias: True') tray = I3Tray() if 'ApplyBaseSimulationBias' in cfg and cfg['ApplyBaseSimulationBias']: n_services = 3 else: n_services = 2 random_services, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=n_services, use_gslrng=cfg['random_service_use_gslrng'], ) random_service, random_service_prop = random_services[:2] tray.context['I3RandomService'] = random_service tray.AddModule("I3InfiniteSource", "TheSource", Prefix=cfg['gcd'], Stream=icetray.I3Frame.DAQ) tray.AddSegment(segments.GenerateNeutrinos, "GenerateNeutrinos", RunID=run_number, RandomService=random_service, NumEvents=cfg['n_events_per_run'], FromEnergy=cfg['e_min'] * icetray.I3Units.GeV, ToEnergy=cfg['e_max'] * icetray.I3Units.GeV, ZenithRange=[ cfg['zenith_min'] * icetray.I3Units.deg, cfg['zenith_max'] * icetray.I3Units.deg ], AzimuthRange=[ cfg['azimuth_min'] * icetray.I3Units.deg, cfg['azimuth_max'] * icetray.I3Units.deg ], **cfg['additional_GenerateNeutrinos_settings']) # propagate muons if config exists in config # Note: Snowstorm may perform muon propagation internally if 'muon_propagation_config' in cfg: tray.AddSegment(segments.PropagateMuons, 'propagate_muons', RandomService=random_service_prop, **cfg['muon_propagation_config']) else: # In this case we are not propagating the I3MCTree yet, but # are letting this be done by snowstorm propagation # We need to add a key named 'I3MCTree', since snowstorm expects this # It will propagate the particles for us. tray.AddModule('Rename', keys=['I3MCTree_preMuonProp', 'I3MCTree']) # Bias simulation if desired if 'ApplyBaseSimulationBias' in cfg and cfg['ApplyBaseSimulationBias']: tray.AddModule(BaseSimulationBias, 'BaseSimulationBias', random_service=random_services[2], **cfg['BaseSimulationBiasSettings']) if cfg['distance_splits'] is not None: import dom_distance_cut as dom_cut click.echo('Oversizestreams') stream_objects = dom_cut.generate_stream_object( cut_distances=cfg['distance_splits'], dom_limits=cfg['threshold_doms'], oversize_factors=cfg['oversize_factors']) tray.AddModule(dom_cut.OversizeSplitterNSplits, "OversizeSplitterNSplits", thresholds=cfg['distance_splits'], thresholds_doms=cfg['threshold_doms'], oversize_factors=cfg['oversize_factors'], simulaton_type=cfg['neutrino_flavor'].lower()) for stream_i in stream_objects: outfile_i = stream_i.transform_filepath(outfile) click.echo('\t{}'.format(stream_i)) click.echo('\tOutfile: {}'.format(outfile_i)) tray.AddModule("I3Writer", "writer_{}".format(stream_i.stream_name), Filename=outfile_i, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ], If=stream_i) else: click.echo('Output: {}'.format(outfile)) tray.AddModule("I3Writer", "writer", Filename=outfile, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ]) click.echo('Scratch: {}'.format(scratch)) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: cfg = yaml.load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace(' ', '0') click.echo('NEvents: {}'.format(cfg['n_events_per_run'])) click.echo('EMin: {}'.format(cfg['e_min'])) click.echo('EMax: {}'.format(cfg['e_max'])) click.echo('Gamma: {}'.format(cfg['gamma'])) click.echo('ZenithMin: {}'.format(cfg['zenith_min'])) click.echo('ZenithMax: {}'.format(cfg['zenith_max'])) click.echo('AzimuthMin: {}'.format(cfg['azimuth_min'])) click.echo('AzimuthMax: {}'.format(cfg['azimuth_max'])) if cfg['neutrino_flavor'] is None: click.echo('NeutrinoTypes: {}'.format(cfg['neutrino_types'])) click.echo('PrimaryTypeRatio: {}'.format(cfg['primary_type_ratio'])) else: click.echo('NeutrinoFlavor: {}'.format(cfg['neutrino_flavor'])) click.echo('CrossSections: {}'.format(cfg['cross_sections'])) if not cfg['cross_sections_path'] is None: click.echo('CrossSectionsPath: {}'.format(cfg['cross_sections_path'])) tray = I3Tray() random_services, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=2) random_service, random_service_prop = random_services tray.context['I3RandomService'] = random_service tray.AddModule("I3InfiniteSource", "TheSource", Prefix=cfg['gcd'], Stream=icetray.I3Frame.DAQ) tray.AddSegment( segments.GenerateNeutrinos, "GenerateNeutrinos", RandomService=random_service, NumEvents=cfg['n_events_per_run'], SimMode=cfg['simulation_mode'], VTXGenMode=cfg['vertex_generation_mode'], InjectionMode=cfg['injection_mode'], CylinderParams=cfg['cylinder_params'], AutoExtendMuonVolume=cfg['auto_extend_muon_volume'], Flavor=cfg['neutrino_flavor'], # NuTypes = cfg['neutrino_types'], # Only in newer simprod versions # PrimaryTypeRatio = cfg['primary_type_ratio'], # Only in newer simprod versions GammaIndex=cfg['gamma'], FromEnergy=cfg['e_min'] * icetray.I3Units.GeV, ToEnergy=cfg['e_max'] * icetray.I3Units.GeV, ZenithRange=[ cfg['zenith_min'] * icetray.I3Units.deg, cfg['zenith_max'] * icetray.I3Units.deg ], AzimuthRange=[ cfg['azimuth_min'] * icetray.I3Units.deg, cfg['azimuth_max'] * icetray.I3Units.deg ], # UseDifferentialXsection = cfg['use_diff_cross_section'], # Only in newer simprod versions CrossSections=cfg['cross_sections'], CrossSectionsPath=cfg['cross_sections_path'], # ZenithSamplingMode = cfg['zenith_sampling_mode'], # Only in newer simprod versions ) tray.AddSegment(segments.PropagateMuons, "PropagateMuons", RandomService=random_service_prop, **cfg['muon_propagation_config']) if cfg['distance_splits'] is not None: import dom_distance_cut as dom_cut click.echo('Oversizestreams') stream_objects = dom_cut.generate_stream_object( cut_distances=cfg['distance_splits'], dom_limits=cfg['threshold_doms'], oversize_factors=cfg['oversize_factors']) tray.AddModule(dom_cut.OversizeSplitterNSplits, "OversizeSplitterNSplits", thresholds=cfg['distance_splits'], thresholds_doms=cfg['threshold_doms'], oversize_factors=cfg['oversize_factors'], simulaton_type=cfg['neutrino_flavor'].lower()) for stream_i in stream_objects: outfile_i = stream_i.transform_filepath(outfile) click.echo('\t{}'.format(stream_i)) click.echo('\tOutfile: {}'.format(outfile_i)) tray.AddModule("I3Writer", "writer_{}".format(stream_i.stream_name), Filename=outfile_i, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ], If=stream_i) else: click.echo('Output: {}'.format(outfile)) tray.AddModule("I3Writer", "writer", Filename=outfile, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ]) click.echo('Scratch: {}'.format(scratch)) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: cfg = yaml.load(stream, Loader=yaml.Loader) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace(' ', '0') if cfg['distance_splits'] is not None: click.echo('SplittingDistances: {}'.format(cfg['distance_splits'])) click.echo('Oversizefactors: {}'.format(cfg['oversize_factors'])) click.echo('NEvents: {}'.format(cfg['n_events_per_run'])) click.echo('EMin: {}'.format(cfg['e_min'])) click.echo('EMax: {}'.format(cfg['e_max'])) click.echo('EBreak: {}'.format(cfg['muongun_e_break'])) click.echo('Gamma: {}'.format(cfg['gamma'])) click.echo('ZenithMin: {}'.format(cfg['zenith_min'])) click.echo('ZenithMax: {}'.format(cfg['zenith_max'])) # create convex hull if 'use_convex_hull' in cfg and cfg['use_convex_hull']: # hardcode icecube corner points # ToDo: read from geometry file points = [ [-570.90002441, -125.13999939, 501], # string 31 [-256.14001465, -521.08001709, 501], # string 1 [361., -422.82998657, 501], # string 6 [576.36999512, 170.91999817, 501], # string 50 [338.44000244, 463.72000122, 501], # string 74 [101.04000092, 412.79000854, 501], # string 72 [22.11000061, 509.5, 501], # string 78 [-347.88000488, 451.51998901, 501], # string 75 [-570.90002441, -125.13999939, -502], # string 31 [-256.14001465, -521.08001709, -502], # string 1 [361., -422.82998657, -502], # string 6 [576.36999512, 170.91999817, -502], # string 50 [338.44000244, 463.72000122, -502], # string 74 [101.04000092, 412.79000854, -502], # string 72 [22.11000061, 509.5, -502], # string 78 [-347.88000488, 451.51998901, -502], # string 75 ] convex_hull = ConvexHull(points) else: convex_hull = None if 'extend_past_hull' not in cfg: cfg['extend_past_hull'] = 0.0 random_services, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=2) random_service, random_service_prop = random_services # create muon muon = create_muon( azimuth_range=[cfg['azimuth_min'], cfg['azimuth_max']], zenith_range=[cfg['zenith_min'], cfg['zenith_max']], energy_range=[cfg['e_min'], cfg['e_max']], anchor_time_range=cfg['anchor_time_range'], anchor_x_range=cfg['anchor_x_range'], anchor_y_range=cfg['anchor_y_range'], anchor_z_range=cfg['anchor_z_range'], length_to_go_back=cfg['length_to_go_back'], convex_hull=convex_hull, extend_past_hull=cfg['extend_past_hull'], random_service=random_services[0], ) tray = I3Tray() tray.context['I3RandomService'] = random_service tray.AddModule("I3InfiniteSource", "TheSource", Prefix=cfg['gcd'], Stream=icetray.I3Frame.DAQ) if cfg['MuonGenerator'] == 'MuonGunSinglemuons': tray.AddSegment(segments.GenerateSingleMuons, "GenerateCosmicRayMuons", NumEvents=cfg['n_events_per_run'], FromEnergy=cfg['e_min'] * icetray.I3Units.GeV, ToEnergy=cfg['e_max'] * icetray.I3Units.GeV, BreakEnergy=cfg['muongun_e_break'] * icetray.I3Units.GeV, GammaIndex=cfg['gamma'], ZenithRange=[ cfg['zenith_min'] * icetray.I3Units.deg, cfg['zenith_max'] * icetray.I3Units.deg ]) elif cfg['MuonGenerator'] == 'MuonGunGeneral': model = MuonGun.load_model(cfg['muongun_model']) model.flux.min_multiplicity = cfg['muongun_min_multiplicity'] model.flux.max_multiplicity = cfg['muongun_max_multiplicity'] spectrum = MuonGun.OffsetPowerLaw(cfg['gamma'], cfg['e_min'] * icetray.I3Units.GeV, cfg['e_min'] * icetray.I3Units.GeV, cfg['e_max'] * icetray.I3Units.GeV) surface = MuonGun.Cylinder(1600, 800, dataclasses.I3Position(31.25, 19.64, 0)) if cfg['muongun_generator'] == 'energy': scale = MuonGun.BasicSurfaceScalingFunction() scale.SetSideScaling(4., 17266, 3.41, 1.74) scale.SetCapScaling(4., 23710, 3.40, 1.88) generator = MuonGun.EnergyDependentSurfaceInjector( surface, model.flux, spectrum, model.radius, scale) elif cfg['muongun_generator'] == 'static': generator = MuonGun.StaticSurfaceInjector(surface, model.flux, spectrum, model.radius) elif cfg['muongun_generator'] == 'floodlight': generator = MuonGun.Floodlight( surface=surface, energyGenerator=spectrum, cosMin=cfg['muongun_floodlight_min_cos'], cosMax=cfg['muongun_floodlight_max_cos'], ) else: err_msg = 'MuonGun generator {} is not known.' err_msg += " Must be 'energy','static' or 'floodlight" raise ValueError(err_msg.format(cfg['muongun_generator'])) tray.Add(MuonGun.segments.GenerateBundles, 'MuonGenerator', Generator=generator, NEvents=cfg['n_events_per_run'], GCDFile=cfg['gcd']) tray.Add("Rename", keys=["I3MCTree", "I3MCTree_preMuonProp"]) elif cfg['MuonGenerator'] == 'MuonResimulation': tray.AddModule(ParticleMultiplier, 'make_particles', num_events=cfg['n_events_per_run'], primary=muon) else: err_msg = 'MuonGenerator {} is not known.' err_msg += " Must be 'MuonGunSinglemuons','MuonGunGeneral' or 'MuonResimulation" raise ValueError(err_msg.format(cfg['MuonGenerator'])) # -------------------------------------- # Propagate Muons # -------------------------------------- tray.AddSegment(segments.PropagateMuons, "PropagateMuons", RandomService=random_service_prop, **cfg['muon_propagation_config']) # -------------------------------------- # Distance Splits # -------------------------------------- if cfg['distance_splits'] is not None: click.echo('SplittingDistance: {}'.format(cfg['distance_splits'])) distance_splits = np.atleast_1d(cfg['distance_splits']) dom_limits = np.atleast_1d(cfg['threshold_doms']) if len(dom_limits) == 1: dom_limits = np.ones_like(distance_splits) * cfg['threshold_doms'] oversize_factors = np.atleast_1d(cfg['oversize_factors']) order = np.argsort(distance_splits) distance_splits = distance_splits[order] dom_limits = dom_limits[order] oversize_factors = oversize_factors[order] stream_objects = generate_stream_object(distance_splits, dom_limits, oversize_factors) tray.AddModule(OversizeSplitterNSplits, "OversizeSplitterNSplits", thresholds=distance_splits, thresholds_doms=dom_limits, oversize_factors=oversize_factors) for stream_i in stream_objects: outfile_i = stream_i.transform_filepath(outfile) tray.AddModule("I3Writer", "writer_{}".format(stream_i.stream_name), Filename=outfile_i, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ], If=stream_i) click.echo('Output ({}): {}'.format(stream_i.stream_name, outfile_i)) else: click.echo('Output: {}'.format(outfile)) tray.AddModule("I3Writer", "writer", Filename=outfile, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ]) click.echo('Scratch: {}'.format(scratch)) tray.Execute() del tray
def main(config_file, run_number, scratch): with open(config_file, 'r') as stream: if int(yaml.__version__[0]) < 5: # backwards compatibility for yaml versions before version 5 cfg = yaml.load(stream) else: cfg = yaml.full_load(stream) if 'dictitems' in cfg.keys(): cfg = cfg['dictitems'] cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) infile = cfg['infile_pattern'].format(**cfg) infile = infile.replace(' ', '0') infile = infile.replace('Level0.{}'.format(cfg['previous_step']), 'Level0.{}'.format(cfg['previous_step'] % 10)) infile = infile.replace('2012_pass2', '2012') if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace('Level0.{}'.format(cfg['step']), 'Level0.{}'.format(cfg['step'] % 10)) outfile = outfile.replace(' ', '0') outfile = outfile.replace('2012_pass2', '2012') print('Outfile != $FINAL_OUT clean up for crashed scripts not possible!') _, seed = create_random_services(dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=0) tray = I3Tray() tray.AddModule('I3Reader', 'reader', FilenameList=[cfg['gcd_2012'], infile], SkipKeys=[ 'I3DST11', 'I3SuperDST', 'I3VEMCalData', 'PoleMuonLlhFit', 'PoleMuonLlhFitCutsFirstPulseCuts', 'PoleMuonLlhFitFitParams', 'CramerRaoPoleL2IpdfGConvolute_2itParams', 'CramerRaoPoleL2MPEFitParams', 'PoleL2IpdfGConvolute_2it', 'PoleL2IpdfGConvolute_2itFitParams', 'PoleL2MPEFit', 'PoleL2MPEFitCuts', 'PoleL2MPEFitFitParams', 'PoleL2MPEFitMuE', ]) class SkipSFrames(icetray.I3ConditionalModule): S_stream = icetray.I3Frame.Stream('S') def __init__(self, context): icetray.I3ConditionalModule.__init__(self, context) def Configure(self): self.Register(self.S_stream, self.SFrame) def SFrame(self, frame): pass # tray.AddModule(SkipSFrames, # "Skip I Frames") def check_driving_time(frame): if 'DrivingTime' not in frame: frame['DrivingTime'] = dataclasses.I3Time( frame['I3EventHeader'].start_time) return True tray.AddModule(check_driving_time, 'DrivingTimeCheck', Streams=[icetray.I3Frame.DAQ]) # move that old filterMask out of the way tray.AddModule("Rename", "filtermaskmover", Keys=["FilterMask", "OrigFilterMask"]) if cfg['L1_2012_qify']: tray.AddModule("QConverter", "qify", WritePFrame=False) def MissingITCheck(frame): #print "Fixing IceTop RO" if "IceTopRawData" not in frame: itrd = dataclasses.I3DOMLaunchSeriesMap() frame["IceTopRawData"] = itrd tray.AddModule(MissingITCheck, 'L1_AddIceTopPulses', Streams=[icetray.I3Frame.DAQ]) if cfg['L1_2012_retrigger']: # some cleanup first tray.AddModule("Delete", "delete_triggerHierarchy", Keys=["I3TriggerHierarchy", "TimeShift"]) gcd_file = dataio.I3File(cfg['gcd_2012']) tray.AddSegment(trigger_sim.TriggerSim, "trig", gcd_file=gcd_file) tray.AddSegment(jeb_filter_2012.BaseProcessing, "BaseProc", pulses=filter_globals.CleanedMuonPulses, decode=False, simulation=True, DomLauncher=(not cfg['L1_2012_dom_simulator'])) tray.AddSegment(jeb_filter_2012.MuonFilter, "MuonFilter", pulses=filter_globals.CleanedMuonPulses, If=which_split(split_name=filter_globals.InIceSplitter)) tray.AddSegment(jeb_filter_2012.CascadeFilter, "CascadeFilter", pulses=filter_globals.CleanedMuonPulses, muon_llhfit_name=filter_globals.muon_llhfit, If=which_split(split_name=filter_globals.InIceSplitter)) tray.AddSegment(jeb_filter_2012.FSSFilter, "FSSFilter", pulses=filter_globals.SplitUncleanedInIcePulses, If=which_split(split_name=filter_globals.InIceSplitter)) tray.AddSegment(jeb_filter_2012.LowUpFilter, "LowUpFilter", If=which_split(split_name=filter_globals.InIceSplitter)) tray.AddSegment(jeb_filter_2012.ShadowFilter, "ShawdowFilters", If=which_split(split_name=filter_globals.InIceSplitter)) # use the PID as a seed. Good enough? tray.AddSegment(jeb_filter_2012.GCFilter, "GCFilter", mcseed=seed, If=which_split(split_name=filter_globals.InIceSplitter)) tray.AddSegment(jeb_filter_2012.VEFFilter, "VEFFilter", pulses=filter_globals.CleanedMuonPulses, If=which_split(split_name=filter_globals.InIceSplitter)) if PHOTONICS_DIR is not None: photonicstabledirmu = os.path.join(PHOTONICS_DIR, 'SPICE1') photonicsdriverfilemu = os.path.join('driverfiles', 'mu_photorec.list') else: photonicstabledirmu = None photonicsdriverfilemu = None tray.AddSegment(jeb_filter_2012.OnlineL2Filter, "OnlineL2", pulses=filter_globals.CleanedMuonPulses, llhfit_name=filter_globals.muon_llhfit, improved_linefit=True, paraboloid=False, PhotonicsTabledirMu=photonicstabledirmu, PhotonicsDriverfileMu_Spice1=photonicsdriverfilemu, If=which_split(split_name=filter_globals.InIceSplitter)) tray.AddSegment(jeb_filter_2012.DeepCoreFilter, "DeepCoreFilter", pulses=filter_globals.SplitUncleanedInIcePulses, If=which_split(split_name=filter_globals.InIceSplitter)) tray.AddSegment(jeb_filter_2012.EHEFilter, "EHEFilter", If=which_split(split_name=filter_globals.NullSplitter)) tray.AddSegment(jeb_filter_2012.MinBiasFilters, "MinBias", If=which_split(split_name=filter_globals.NullSplitter)) tray.AddSegment(jeb_filter_2012.SlopFilters, "SLOP", If=which_split(split_name=filter_globals.NullSplitter)) tray.AddSegment(jeb_filter_2012.FixedRateTrigFilter, "FixedRate", If=which_split(split_name=filter_globals.NullSplitter)) tray.AddSegment(jeb_filter_2012.CosmicRayFilter, "CosmicRayFilter", pulseMask=filter_globals.SplitUncleanedITPulses, If=which_split(split_name=filter_globals.IceTopSplitter)) tray.AddSegment(jeb_filter_2012.DST, "DSTFilter", dstname="I3DST12", pulses=filter_globals.CleanedMuonPulses, If=which_split(split_name=filter_globals.InIceSplitter)) # make random service filter_mask_randoms = phys_services.I3GSLRandomService(seed) # override MinBias Prescale filterconfigs = filter_globals.filter_pairs + filter_globals.sdst_pairs if cfg['L1_min_bias_prescale'] is not None: for i, filtertuple in enumerate(filterconfigs): if filtertuple[0] == filter_globals.FilterMinBias: del filterconfigs[i] filterconfigs.append( (filtertuple[0], cfg['L1_min_bias_prescale'])) break click.echo(filterconfigs) # Generate filter Masks for all P frames tray.AddModule(filter_tools.FilterMaskMaker, "MakeFilterMasks", OutputMaskName=filter_globals.filter_mask, FilterConfigs=filterconfigs, RandomService=filter_mask_randoms) # Merge the FilterMasks tray.AddModule("OrPframeFilterMasks", "make_q_filtermask", InputName=filter_globals.filter_mask, OutputName=filter_globals.qfilter_mask) #Q+P frame specific keep module needs to go first, as KeepFromSubstram #will rename things, let's rename post keep. def is_Q(frame): return frame.Stop == frame.DAQ tray.AddModule( "Keep", "keep_before_merge", keys=filter_globals.q_frame_keeps + [ 'InIceDSTPulses', # keep DST pulse masks 'IceTopDSTPulses', 'CalibratedWaveformRange', # keep calibration info 'UncleanedInIcePulsesTimeRange', 'SplitUncleanedInIcePulsesTimeRange', 'SplitUncleanedInIceDSTPulsesTimeRange', 'CalibrationErrata', 'SaturationWindows', 'InIceRawData', # keep raw data for now 'IceTopRawData', 'CorsikaWeightMap', # sim keys 'I3MCWeightDict', 'MCHitSeriesMap', 'MMCTrackList', 'I3MCTree', 'I3LinearizedMCTree', 'MCPrimary', 'MCPrimaryInfo', 'TimeShift', # the time shift amount 'WIMP_params', # Wimp-sim 'SimTrimmer', # for SimTrimmer flag 'I3MCPESeriesMap', 'I3MCPulseSeriesMap', 'I3MCPulseSeriesMapParticleIDMap', ] + muongun_keys, If=is_Q) tray.AddModule("I3IcePickModule<FilterMaskFilter>", "filterMaskCheckAll", FilterNameList=filter_globals.filter_streams, FilterResultName=filter_globals.qfilter_mask, DecisionName="PassedAnyFilter", DiscardEvents=False, Streams=[icetray.I3Frame.DAQ]) def do_save_just_superdst(frame): if frame.Has("PassedAnyFilter"): if not frame["PassedAnyFilter"].value: return True # <- Event failed to pass any filter. else: return False # <- Event passed some filter else: print "Failed to find key frame Bool!!" return False tray.AddModule( "Keep", "KeepOnlySuperDSTs", keys=filter_globals.keep_nofilterpass + [ 'PassedAnyFilter', 'InIceDSTPulses', 'IceTopDSTPulses', 'SplitUncleanedInIcePulses', 'SplitUncleanedInIcePulsesTimeRange', 'SplitUncleanedInIceDSTPulsesTimeRange', 'CorsikaWeightMap', # sim keys 'I3MCWeightDict', 'MCHitSeriesMap', 'MMCTrackList', 'I3MCTree', 'I3LinearizedMCTree', 'MCPrimary', 'MCPrimaryInfo', 'TimeShift', # the time shift amount 'WIMP_params', # Wimp-sim 'I3MCPESeriesMap', 'I3MCPulseSeriesMap', 'I3MCPulseSeriesMapParticleIDMap', ] + muongun_keys, If=do_save_just_superdst) ## Now clean up the events that not even the SuperDST filters passed on. tray.AddModule("I3IcePickModule<FilterMaskFilter>", "filterMaskCheckSDST", FilterNameList=filter_globals.sdst_streams, FilterResultName=filter_globals.qfilter_mask, DecisionName="PassedKeepSuperDSTOnly", DiscardEvents=False, Streams=[icetray.I3Frame.DAQ]) def dont_save_superdst(frame): if frame.Has("PassedKeepSuperDSTOnly") and frame.Has( "PassedAnyFilter"): if frame["PassedAnyFilter"].value: return False # <- these passed a regular filter, keeper elif not frame["PassedKeepSuperDSTOnly"].value: return True # <- Event failed to pass SDST filter. else: return False # <- Event passed some SDST filter else: print "Failed to find key frame Bool!!" return False # backward compatibility if 'L1_keep_untriggered' in cfg and cfg['L1_keep_untriggered']: discard_substream_and_keys = False else: discard_substream_and_keys = True if discard_substream_and_keys: tray.AddModule("Keep", "KeepOnlyDSTs", keys=filter_globals.keep_dst_only + [ "PassedAnyFilter", "PassedKeepSuperDSTOnly", filter_globals.eventheader ] + muongun_keys, If=dont_save_superdst) ## Frames should now contain only what is needed. now flatten, write/send to server ## Squish P frames back to single Q frame, one for each split: tray.AddModule( "KeepFromSubstream", "null_stream", StreamName=filter_globals.NullSplitter, KeepKeys=filter_globals.null_split_keeps, ) # Keep the P frames for InIce intact #tray.AddModule("KeepFromSubstream","inice_split_stream", # StreamName = filter_globals.InIceSplitter, # KeepKeys = filter_globals.inice_split_keeps + filter_globals.onlinel2filter_keeps, # ) # in_ice_keeps = filter_globals.inice_split_keeps + \ filter_globals.onlinel2filter_keeps in_ice_keeps = in_ice_keeps + [ 'I3EventHeader', 'SplitUncleanedInIcePulses', 'SplitUncleanedInIcePulsesTimeRange', 'SplitUncleanedInIceDSTPulsesTimeRange', 'I3TriggerHierarchy', 'GCFilter_GCFilterMJD' ] tray.AddModule( "Keep", "inice_keeps", keys=in_ice_keeps + muongun_keys, If=which_split(split_name=filter_globals.InIceSplitter), ) tray.AddModule( "KeepFromSubstream", "icetop_split_stream", StreamName=filter_globals.IceTopSplitter, KeepKeys=filter_globals.icetop_split_keeps, ) tray.AddModule("I3IcePickModule<FilterMaskFilter>", "filterMaskCheck", FilterNameList=filter_globals.filters_keeping_allraw, FilterResultName=filter_globals.qfilter_mask, DecisionName="PassedConventional", DiscardEvents=False, Streams=[icetray.I3Frame.DAQ]) ## Clean out the Raw Data when not passing conventional filter def I3RawDataCleaner(frame): if not (('PassedConventional' in frame and frame['PassedConventional'].value == True) or ('SimTrimmer' in frame and frame['SimTrimmer'].value == True)): frame.Delete('InIceRawData') frame.Delete('IceTopRawData') tray.AddModule(I3RawDataCleaner, "CleanErrataForConventional", Streams=[icetray.I3Frame.DAQ]) tray.AddModule("I3Writer", "EventWriter", filename=outfile, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.TrayInfo, icetray.I3Frame.Stream('S') ]) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: cfg = yaml.load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) tray = I3Tray() random_service, random_service_prop, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed']) tray.context['I3RandomService'] = random_service model = MuonGun.load_model(cfg['muongun_model']) model.flux.min_multiplicity = cfg['muongun_min_multiplicity'] model.flux.max_multiplicity = cfg['muongun_max_multiplicity'] spectrum = MuonGun.OffsetPowerLaw( cfg['gamma'], cfg['e_min']*icetray.I3Units.TeV, cfg['e_min']*icetray.I3Units.TeV, cfg['e_max']*icetray.I3Units.TeV) surface = MuonGun.Cylinder(1600, 800, dataclasses.I3Position(31.25, 19.64, 0)) if cfg['muongun_generator'] == 'energy': scale = MuonGun.BasicSurfaceScalingFunction() scale.SetSideScaling(4., 17266, 3.41, 1.74) scale.SetCapScaling(4., 23710, 3.40, 1.88) generator = MuonGun.EnergyDependentSurfaceInjector(surface, model.flux, spectrum, model.radius, scale) elif cfg['muongun_generator'] == 'static': generator = MuonGun.StaticSurfaceInjector(surface, model.flux, spectrum, model.radius) elif cfg['muongun_generator'] =='floodlight': generator = MuonGun.Floodlight(surface = surface, energyGenerator=spectrum, cosMin=cfg['muongun_floodlight_min_cos'], cosMax=cfg['muongun_floodlight_max_cos'], ) else: err_msg = 'MuonGun generator {} is not known.' err_msg += " Must be 'energy','static' or 'floodlight" raise ValueError(err_msg.format(cfg['muongun_generator'])) tray.Add(MuonGun.segments.GenerateBundles, 'MuonGenerator', Generator=generator, NEvents=cfg['n_events_per_run'], GCDFile=cfg['gcd']) tray.Add("Rename", keys=["I3MCTree", "I3MCTree_preMuonProp"]) tray.AddSegment( segments.PropagateMuons, "PropagateMuons", RandomService=random_service_prop) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace(' ', '0') outfile = outfile.replace('.bz2', '') tray.AddModule("I3Writer", "writer", Filename=outfile, Streams=[icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M')]) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: if int(yaml.__version__[0]) < 5: # backwards compatibility for yaml versions before version 5 cfg = yaml.load(stream) else: cfg = yaml.full_load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) tray = I3Tray() random_services, _ = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=2) random_service, random_service_prop = random_services tray.context['I3RandomService'] = random_service model = MuonGun.load_model(cfg['muongun_model']) model.flux.min_multiplicity = cfg['muongun_min_multiplicity'] model.flux.max_multiplicity = cfg['muongun_max_multiplicity'] spectrum = MuonGun.OffsetPowerLaw(cfg['gamma'], cfg['e_min'] * icetray.I3Units.GeV, cfg['e_min'] * icetray.I3Units.GeV, cfg['e_max'] * icetray.I3Units.GeV) surface = MuonGun.Cylinder(1600, 800, dataclasses.I3Position(31.25, 19.64, 0)) if cfg['muongun_generator'] == 'energy': scale = MuonGun.BasicSurfaceScalingFunction() scale.SetSideScaling(4., 17266, 3.41, 1.74) scale.SetCapScaling(4., 23710, 3.40, 1.88) generator = MuonGun.EnergyDependentSurfaceInjector( surface, model.flux, spectrum, model.radius, scale) elif cfg['muongun_generator'] == 'static': generator = MuonGun.StaticSurfaceInjector(surface, model.flux, spectrum, model.radius) elif cfg['muongun_generator'] == 'floodlight': generator = MuonGun.Floodlight( surface=surface, energyGenerator=spectrum, cosMin=cfg['muongun_floodlight_min_cos'], cosMax=cfg['muongun_floodlight_max_cos'], ) else: err_msg = 'MuonGun generator {} is not known.' err_msg += " Must be 'energy','static' or 'floodlight" raise ValueError(err_msg.format(cfg['muongun_generator'])) tray.Add(MuonGun.segments.GenerateBundles, 'MuonGenerator', Generator=generator, NEvents=cfg['n_events_per_run'], GCDFile=cfg['gcd']) tray.Add("Rename", keys=["I3MCTree", "I3MCTree_preMuonProp"]) tray.AddSegment(segments.PropagateMuons, "PropagateMuons", RandomService=random_service_prop, **cfg['muon_propagation_config']) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace(' ', '0') if cfg['distance_splits'] is not None: click.echo('SplittingDistance: {}'.format(cfg['distance_splits'])) distance_splits = np.atleast_1d(cfg['distance_splits']) dom_limits = np.atleast_1d(cfg['threshold_doms']) if len(dom_limits) == 1: dom_limits = np.ones_like(distance_splits) * cfg['threshold_doms'] oversize_factors = np.atleast_1d(cfg['oversize_factors']) order = np.argsort(distance_splits) distance_splits = distance_splits[order] dom_limits = dom_limits[order] oversize_factors = oversize_factors[order] stream_objects = generate_stream_object(distance_splits, dom_limits, oversize_factors) tray.AddModule(OversizeSplitterNSplits, "OversizeSplitterNSplits", thresholds=distance_splits, thresholds_doms=dom_limits, oversize_factors=oversize_factors) for stream_i in stream_objects: outfile_i = stream_i.transform_filepath(outfile) tray.AddModule("I3Writer", "writer_{}".format(stream_i.stream_name), Filename=outfile_i, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ], If=stream_i) click.echo('Output ({}): {}'.format(stream_i.stream_name, outfile_i)) else: click.echo('Output: {}'.format(outfile)) tray.AddModule("I3Writer", "writer", Filename=outfile, Streams=[ icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.Stream('S'), icetray.I3Frame.Stream('M') ]) click.echo('Scratch: {}'.format(scratch)) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()
def main(cfg, run_number, scratch): with open(cfg, 'r') as stream: cfg = yaml.full_load(stream) cfg['run_number'] = run_number cfg['run_folder'] = get_run_folder(run_number) infile = cfg['infile_pattern'].format(**cfg) infile = infile.replace(' ', '0') infile = infile.replace('Level0.{}'.format(cfg['previous_step']), 'Level0.{}'.format(cfg['previous_step'] % 10)) if scratch: outfile = cfg['scratchfile_pattern'].format(**cfg) else: outfile = cfg['outfile_pattern'].format(**cfg) outfile = outfile.replace('Level0.{}'.format(cfg['step']), 'Level0.{}'.format(cfg['step'] % 10)) outfile = outfile.replace(' ', '0') outfile = outfile.replace('2012_pass2', 'pass2') print('Outfile != $FINAL_OUT clean up for crashed scripts not possible!') tray = I3Tray() tray.context['I3FileStager'] = dataio.get_stagers() random_services, run_id = create_random_services( dataset_number=cfg['dataset_number'], run_number=cfg['run_number'], seed=cfg['seed'], n_services=1, use_gslrng=cfg['random_service_use_gslrng']) random_service = random_services[0] tray.context['I3RandomService'] = random_service tray.Add('I3Reader', FilenameList=[cfg['gcd_pass2'], infile]) """ Perform Detector simulation: https://code.icecube.wisc.edu/projects/icecube/browser/IceCube/ meta-projects/combo/stable/simprod-scripts/python/segments/ DetectorSim.py """ # Combine MCPEs from both detectors if cfg['det_is_genie_simulation']: tray.Add("Rename", Keys=[MCPE_SERIES_MAP, 'GenieMCPEs']) tray.Add("I3CombineMCPE", InputResponses=["GenieMCPEs", "BackgroundMCPEs"], OutputResponse=MCPE_SERIES_MAP) tray.Add("Delete", Keys=['BackgroundMCPEs', 'GenieMCPEs']) if cfg['det_is_icetop_simulation']: tray.Add("Rename", Keys=[MCPE_SERIES_MAP, 'InIceMCPEs']) tray.Add("I3CombineMCPE", InputResponses=["IceTopMCPEs", "InIceMCPEs"], OutputResponse=MCPE_SERIES_MAP) tray.Add("Delete", Keys=['InIceMCPEs', 'IceTopMCPEs']) # Sample a different efficiency sample_eff = cfg['det_dom_eff_resmapling_sample_efficiency'] generated_eff = cfg['det_dom_eff_resmapling_generated_efficiency'] if sample_eff > 0.0: if sample_eff > generated_eff: msg = 'Cannot upscale from GeneratedEfficiency %s to ' msg += 'SampleEfficiency %s' icecube.icetray.logging.log_fatal( msg % (sample_eff, generated_eff)) tray.AddSegment(segments.MultiDomEffSample, "resample", GeneratedEfficiency=generated_eff, SampleEfficiencies=[sample_eff], InputSeriesName=MCPE_SERIES_MAP, DeleteOriginalSeries=True, OverwriteOriginalSeries=True) if run_number < cfg['det_keep_all_upto']: cfg['det_keep_mc_hits'] = True cfg['det_keep_propagated_mc_tree'] = True cfg['det_keep_mc_pulses'] = True tray.AddSegment(segments.DetectorSim, "DetectorSim", RandomService='I3RandomService', RunID=run_id, GCDFile=cfg['gcd_pass2'], KeepMCHits=cfg['det_keep_mc_hits'], KeepPropagatedMCTree=cfg['det_keep_propagated_mc_tree'], KeepMCPulses=cfg['det_keep_mc_pulses'], SkipNoiseGenerator=cfg['det_skip_noise_generation'], LowMem=cfg['det_low_mem'], InputPESeriesMapName=MCPE_SERIES_MAP, BeaconLaunches=cfg['det_add_beacon_launches'], FilterTrigger=cfg['det_filter_trigger'], TimeShiftSkipKeys=[ "SnowstormParameterRanges", "SnowstormParameters", "SnowstormParametrizations", "SnowstormProposalDistribution", "WavelengthAcceptance", "WavelengthGenerationBias", "LeptonInjectorProperties", "EventProperties", "MediumProperties", ], ) if not cfg['det_keep_mc_pulses']: tray.Add("Delete", Keys=['I3MCPulseSeriesMapPrimaryIDMap']) if not cfg['det_keep_mc_hits']: tray.Add("Delete", Keys=['I3MCPESeriesMapParticleIDMap']) if cfg['det_remove_keys_from_m_frame']: tray.Add("Delete", Keys=cfg['det_remove_keys_from_m_frame']) if cfg['det_convert_to_linear_tree']: tray.AddModule(segments.ConvertToLinearizedMCTree, "lineartree", streams=[icetray.I3Frame.DAQ]) tray.AddModule("I3Writer", "EventWriter", filename=outfile, Streams=[icetray.I3Frame.DAQ, icetray.I3Frame.Physics, icetray.I3Frame.TrayInfo, icetray.I3Frame.Simulation, icetray.I3Frame.Stream('m'), icetray.I3Frame.Stream('M')]) tray.AddModule("TrashCan", "the can") tray.Execute() tray.Finish()