def test_root_output(self): # Get an event mypax = core.Processor(config_names='XENON100', config_dict={ 'pax': { 'events_to_process': [0], 'output': 'Dummy.DummyOutput', 'encoder_plugin': None } }) mypax.run() event = mypax.get_plugin_by_name('DummyOutput').last_event del mypax # Write same event to ROOT mypax = core.Processor(config_names='XENON100', config_dict={ 'pax': { 'events_to_process': [0], 'output_name': 'test_root_output' } }) mypax.run() del mypax self.assertTrue(os.path.exists('test_root_output.root')) self.assertTrue(hasattr(ROOT, 'Peak')) # Can't test event class loading, event class already loaded during writing # ROOTClass.load_pax_event_class_from_root('test_root_output.root') f = ROOT.TFile('test_root_output.root') t = f.Get('tree') t.GetEntry(0) root_event = t.events self.assertEqual(len(root_event.peaks), len(event.peaks)) for i in range(len(event.peaks)): peak = event.peaks[i] root_peak = root_event.peaks[i] self.assertEqual(peak.type, root_peak.type) # 5th or 6th significant figure appears to be different.. float precision difference? self.assertAlmostEqual(peak.area, root_peak.area, delta=0.0001 * max(1, peak.area)) # Check area per channel self.assertAlmostEqual(peak.area, peak.area_per_channel.sum()) self.assertAlmostEqual(peak.area, sum(root_peak.area_per_channel), delta=0.0001 * max(1, peak.area)) np.testing.assert_array_almost_equal( peak.area_per_channel, np.array(list(root_peak.area_per_channel)), decimal=4)
def test_dummy_transform_plugin(self): mypax = core.Processor(config_dict={ 'pax': { 'plugin_group_names': ['bla'], 'bla': 'Dummy.DummyTransform' } }, just_testing=True) self.assertIsInstance(mypax, core.Processor) self.assertIsInstance(mypax.action_plugins[0], plugin.TransformPlugin)
def test_dummy_input_plugin(self): mypax = core.Processor(config_dict={ 'pax': { 'plugin_group_names': ['input'], 'input': 'Dummy.DummyInput' } }, just_testing=True) self.assertIsInstance(mypax, core.Processor) self.assertIsInstance(mypax.input_plugin, plugin.InputPlugin)
def setUp(self): self.pax = core.Processor(config_names='XENON100', just_testing=True, config_dict={ 'pax': { 'plugin_group_names': ['test'], 'test': 'MaxPMT.PosRecMaxPMT' } }) self.plugin = self.pax.get_plugin_by_name('PosRecMaxPMT')
def test_get_input_plugin_by_name(self): mypax = core.Processor(config_dict={ 'pax': { 'plugin_group_names': ['input'], 'input': 'Dummy.DummyInput' } }, just_testing=True) pl = mypax.get_plugin_by_name('DummyInput') self.assertIsInstance(pl, plugin.InputPlugin) self.assertEqual(pl.__class__.__name__, 'DummyInput')
def setUp(self): self.pax = core.Processor(config_names='XENON100', just_testing=True, config_dict={ 'pax': { 'plugin_group_names': ['test'], 'test': 'HitpatternSpread.HitpatternSpread' } }) self.plugin = self.pax.get_plugin_by_name('HitpatternSpread')
def setUp(self): # noqa self.pax = core.Processor(config_names='XENON100', just_testing=True, config_dict={ 'pax': { 'plugin_group_names': ['test'], 'test': 'CheckPulses.CheckBoundsAndCount'}, 'CheckPulses.CheckBoundsAndCount': { 'truncate_pulses_partially_outside': True}}) self.plugin = self.pax.get_plugin_by_name('CheckBoundsAndCount') self.baseline = self.pax.config['DEFAULT']['digitizer_reference_baseline']
def test_dummy_output_plugin(self): mypax = core.Processor(config_dict={ 'pax': { 'plugin_group_names': ['output'], 'encoder_plugin': None, 'output': 'Dummy.DummyOutput' } }, just_testing=True) self.assertIsInstance(mypax, core.Processor) self.assertIsInstance(mypax.action_plugins[0], plugin.OutputPlugin)
def setUp(self): self.pax = core.Processor(config_names='XENON1T', just_testing=True, config_dict={ 'pax': { 'plugin_group_names': ['test'], 'look_for_config_in_runs_db': False, 'test': 'NeuralNet.PosRecNeuralNet' } }) self.plugin = self.pax.get_plugin_by_name('PosRecNeuralNet')
def setUp(self): self.pax = core.Processor( config_names='XENON100', just_testing=True, config_dict={ 'pax': { 'plugin_group_names': ['test'], 'test': 'CheckPulses.ConcatenateAdjacentPulses' } }) self.plugin = self.pax.get_plugin_by_name('ConcatenateAdjacentPulses')
def test_pax_config_string(self): """ Similar, but using an almost-empty config string """ mypax = core.Processor(config_string="[pax]", just_testing=True) self.assertIsInstance(mypax, core.Processor) # Make sure the configuration is actually empty, and a default config did not sneakily get loaded... # Note plugin_group_names gets autoset during config init self.assertEqual(mypax.config, { 'DEFAULT': {}, 'pax': { 'plugin_group_names': [] } })
def test_hitfinder(self): # Integration test for the hitfinder self.pax = core.Processor( config_names='XENON100', just_testing=True, config_dict={ 'pax': { 'plugin_group_names': ['test'], 'encoder_plugin': None, 'decoder_plugin': None, 'test': ['PulseProperties.PulseProperties', 'HitFinder.FindHits'] }, 'HitFinder.FindHits': { 'left_extension': 0, 'right_extension': 0 } }) for test_w, hit_bounds, pulse_min, pulse_max in ( # Keep in mind the hitfinder flips the pulse... [np.zeros(100), [], 0, 0], [np.ones(100), [], 0, 0], [-3 * np.ones(100), [], 0, 0], [self.peak_at(70, amplitude=-100, width=4), [[70, 73]], 0, 100], [ self.peak_at(70, amplitude=-100, width=4) + self.peak_at(80, amplitude=10, width=4), [[70, 73]], -10, 100 ], [ self.peak_at(70, amplitude=-100, width=4) + self.peak_at(80, amplitude=-100, width=4), [[70, 73], [80, 83]], 0, 100 ], ): e = datastructure.Event( n_channels=self.pax.config['DEFAULT']['n_channels'], start_time=0, sample_duration=self.pax.config['DEFAULT']['sample_duration'], stop_time=int(1e6), pulses=[ dict(left=0, raw_data=np.array(test_w).astype(np.int16), channel=1) ]) e = self.pax.process_event(e) self.assertEqual(hit_bounds, [[hit['left'], hit['right']] for hit in e.all_hits]) self.assertEqual(pulse_min, e.pulses[0].minimum) self.assertEqual(pulse_max, e.pulses[0].maximum) delattr(self, 'pax')
def test_process_single_xed_event(self): """ Process the first event from the XED file. """ config = { 'pax': { 'events_to_process': [0], 'encoder_plugin': None, 'output': 'Dummy.DummyOutput' } } mypax = core.Processor(config_names='XENON100', config_dict=config) mypax.run()
def test_simulator(self): """ Process the events in dummy_waveforms.csv """ mypax = core.Processor(config_names=['XENON100', 'Simulation'], config_dict={ 'pax': { 'encoder_plugin': None, 'output': 'Dummy.DummyOutput' } }) mypax.run() pl = mypax.get_plugin_by_name('DummyOutput') self.assertIsInstance(pl, plugin.OutputPlugin)
def test_get_events(self): """ Test getting events from the input plugin """ mypax = core.Processor(config_dict={ 'pax': { 'plugin_group_names': ['input'], 'input': 'Dummy.DummyInput' } }, just_testing=True) self.assertTrue(inspect.isgeneratorfunction(mypax.get_events)) event_generator = mypax.get_events() event = next(event_generator) self.assertIsInstance(event, datastructure.Event)
def test_process_empty_event(self): """ Test processing without processing plugins defined """ mypax = core.Processor(config_dict={ 'pax': { 'plugin_group_names': ['input'], 'input': 'Dummy.DummyInput' } }, just_testing=True) event_generator = mypax.get_events() event = next(event_generator) event = mypax.process_event(event) self.assertIsInstance(event, datastructure.Event)
def test_pax_minimal(self): """ The smallest possible test that actually instantiates the processor. Does not load any plugins or configuration """ mypax = core.Processor(config_dict={'pax': {}}, just_testing=True) self.assertIsInstance(mypax, core.Processor) # Make sure the configuration is actually empty, and a default config did not sneakily get loaded... # Note plugin_group_names gets autoset during config init self.assertEqual(mypax.config, { 'DEFAULT': {}, 'pax': { 'plugin_group_names': [] } })
def raw_data_processor(input_file_or_directory, config_override=None): """Return a raw data processor which reads events from input_file_or_directory config_override can be used to set additional pax options """ if config_override is None: config_override = {} # Add the input name to the config_override # Apply the user overrides, section by section config_override.setdefault('pax', {}) config_override['pax']['input_name'] = input_file_or_directory return core.Processor(config_names=hax.config['experiment'], config_dict=config_override)
def setUp(self): # noqa self.pax = core.Processor(config_names='XENON100', just_testing=True, config_dict={ 'pax': { 'plugin_group_names': ['test'], 'test': 'WeightedSum.PosRecWeightedSum' } }) self.posrec_plugin = self.pax.get_plugin_by_name('PosRecWeightedSum') self.e = Event.empty_event() self.e.peaks.append(Peak({'left': 5, 'right': 9, 'type': 's2'}))
def test_get_plugin_by_name(self): mypax = core.Processor(config_dict={ 'pax': { 'plugin_group_names': ['bla'], 'bla': [ 'Dummy.DummyTransform', 'Dummy.DummyTransform2', 'Dummy.DummyTransform3' ] } }, just_testing=True) self.assertIsInstance(mypax, core.Processor) pl = mypax.get_plugin_by_name('DummyTransform2') self.assertIsInstance(pl, plugin.TransformPlugin) self.assertEqual(pl.__class__.__name__, 'DummyTransform2')
def setUp(self): # noqa self.pax = core.Processor(config_names='XENON100', just_testing=True, config_dict={ 'pax': { 'plugin_group_names': ['test'], 'test': 'BuildInteractions.BuildInteractions' }, 'BuildInteractions.BuildInteractions': { 'pair_n_s2s': 100, 'pair_n_s1s': 100, 's2_pairing_threshold': 101 * (7 + 1) + 1, 'xy_posrec_preference': ['a', 'b'] } }) self.plugin = self.pax.get_plugin_by_name('BuildInteractions')
def setUp(self): self.pax = core.Processor(config_names='XENON100', just_testing=True, config_dict={ 'pax': { 'plugin_group_names': ['test'], 'test': 'ZLE.SoftwareZLE' }, 'ZLE.SoftwareZLE': { 'zle_threshold': 40, 'samples_to_store_before': 50, 'samples_to_store_after': 50, 'max_intervals': 32, 'special_thresholds': {} } }) self.plugin = self.pax.get_plugin_by_name('SoftwareZLE')
def test_process_event_list(self): """ Take a list of event numbers from a file """ with open('temp_eventlist.txt', mode='w') as outfile: outfile.write("0\n7\n") config = { 'pax': { 'event_numbers_file': 'temp_eventlist.txt', 'plugin_group_names': ['input', 'output'], 'encoder_plugin': None, 'output': 'Dummy.DummyOutput' } } mypax = core.Processor(config_names='XENON100', config_dict=config) mypax.run() self.assertEqual( mypax.get_plugin_by_name('DummyOutput').last_event.event_number, 7) os.remove('temp_eventlist.txt')
def test_custom_plugin_location(self): """Tests loading a plugin from a custom location""" tempdir = tempfile.mkdtemp() with open(os.path.join(tempdir, 'temp_plugin_file.py'), mode='w') as outfile: outfile.write(dummy_plugin) mypax = core.Processor(config_dict={ 'pax': { 'plugin_group_names': ['bla'], 'plugin_paths': [tempdir], 'bla': 'temp_plugin_file.FunkyTransform' } }, just_testing=True) self.assertIsInstance(mypax, core.Processor) pl = mypax.get_plugin_by_name('FunkyTransform') self.assertIsInstance(pl, plugin.TransformPlugin) self.assertTrue(hasattr(pl, 'gnork')) shutil.rmtree(tempdir)
def setUp(self): self.pax = core.Processor( config_names='XENON100', just_testing=True, config_dict={ 'pax': { 'plugin_group_names': ['test'], 'test': [ 'WeightedSum.PosRecWeightedSum', 'TopPatternFit.PosRecTopPatternFit' ], 'logging_level': 'debug' } }) self.guess_plugin = self.pax.get_plugin_by_name('PosRecWeightedSum') self.plugin = self.pax.get_plugin_by_name('PosRecTopPatternFit') self.e = Event.empty_event()
def setUp(self): self.pax = core.Processor( config_names='XENON1T', just_testing=True, config_dict={ 'pax': { 'plugin_group_names': ['test'], 'look_for_config_in_runs_db': False, 'test': [ 'WeightedSum.PosRecWeightedSum', 'TopPatternFit.PosRecTopPatternFunctionFit' ], 'logging_level': 'debug' } }) self.guess_plugin = self.pax.get_plugin_by_name('PosRecWeightedSum') self.plugin = self.pax.get_plugin_by_name( 'PosRecTopPatternFunctionFit')
def test_plotting(self): """ Plot the first event from the default XED file """ import matplotlib # Force matplotlib to switch to a non-GUI backend, so the test runs on Travis matplotlib.pyplot.switch_backend('Agg') mypax = core.Processor(config_names='XENON100', config_dict={ 'pax': { 'output': 'Plotting.PlotEventSummary', 'pre_output': [], 'encoder_plugin': None, 'events_to_process': [0], 'output_name': 'plots_test' } }) mypax.run() self.assertTrue(os.path.exists('./plots_test')) self.assertTrue(os.path.exists('./plots_test/000000_000000.png')) shutil.rmtree('plots_test')
def test_process_single_xed_event_olddsp(self): """ Process the first event from the XED file using Xerawdp matching config """ mypax = core.Processor(config_names=['XENON100', 'XerawdpImitation'], config_dict={ 'pax': { 'events_to_process': [0], 'encoder_plugin': None, 'output': 'Dummy.DummyOutput' } }) mypax.run() pl = mypax.get_plugin_by_name('DummyOutput') self.assertIsInstance(pl, plugin.OutputPlugin) e = pl.last_event self.assertIsInstance(e, datastructure.Event) # Check that the peak areas remain the same self.assertEqual([x.area for x in e.peaks], [ 176279.0866616674, 736.576200034518, 611.0961166092862, 129.12023409842166, 88.43269016354068, 16.19189004866498, 430.28177885354137, 1.9494012301013646, 1.52583418095758, 1.5248293965443862, 1.5214431653719354, 1.1431245035453605, 1.119126521198631, 0.8370846398458212, 0.3965132112382404 ])
def pax_to_records(input_filename, samples_per_record=strax.DEFAULT_RECORD_LENGTH, events_per_chunk=10): """Return pulse records array from pax zip input_filename This only works if you have pax installed in your strax environment, which is somewhat tricky. """ # Monkeypatch matplotlib so pax is importable # See https://github.com/XENON1T/pax/pull/734 import matplotlib matplotlib._cntr = None from pax import core # Pax is not a dependency mypax = core.Processor( 'XENON1T', config_dict=dict( pax=dict(look_for_config_in_runs_db=False, plugin_group_names=['input'], encoder_plugin=None, input_name=input_filename), # Fast startup: skip loading big maps WaveformSimulator=dict(s1_light_yield_map='placeholder_map.json', s2_light_yield_map='placeholder_map.json', s1_patterns_file=None, s2_patterns_file=None))) print(f"Starting conversion, {events_per_chunk} evt/chunk") results = [] def finish_results(): nonlocal results records = np.concatenate(results) # In strax data, records are always stored # sorted, baselined and integrated records = strax.sort_by_time(records) strax.baseline(records) strax.integrate(records) print("Returning %d records" % len(records)) results = [] return records for event in mypax.get_events(): event = mypax.process_event(event) if not len(event.pulses): # Triggerless pax data contains many empty events # at the end. With the fixed events per chunk setting # this can lead to empty files, which confuses strax. continue pulse_lengths = np.array([p.length for p in event.pulses]) n_records_tot = records_needed(pulse_lengths, samples_per_record).sum() records = np.zeros(n_records_tot, dtype=strax.record_dtype(samples_per_record)) output_record_index = 0 # Record offset in data for p in event.pulses: n_records = records_needed(p.length, samples_per_record) for rec_i in range(n_records): r = records[output_record_index] r['time'] = (event.start_time + p.left * 10 + rec_i * samples_per_record * 10) r['channel'] = p.channel r['pulse_length'] = p.length r['record_i'] = rec_i r['dt'] = 10 # How much are we storing in this record? if rec_i != n_records - 1: # There's more chunks coming, so we store a full chunk n_store = samples_per_record assert p.length > samples_per_record * (rec_i + 1) else: # Just enough to store the rest of the data # Note it's not p.length % samples_per_record!!! # (that would be zero if we have to store a full record) n_store = p.length - samples_per_record * rec_i assert 0 <= n_store <= samples_per_record r['length'] = n_store offset = rec_i * samples_per_record r['data'][:n_store] = p.raw_data[offset:offset + n_store] output_record_index += 1 results.append(records) if len(results) >= events_per_chunk: yield finish_results() mypax.shutdown() if len(results): y = finish_results() if len(y): yield y
def pax_to_records(input_filename, samples_per_record=strax.DEFAULT_RECORD_LENGTH, events_per_chunk=10): """Return pulse records array from pax zip input_filename""" from pax import core # Pax is not a dependency mypax = core.Processor('XENON1T', config_dict=dict( pax=dict( look_for_config_in_runs_db=False, plugin_group_names=['input'], encoder_plugin=None, input_name=input_filename), # Fast startup: skip loading big maps WaveformSimulator=dict( s1_light_yield_map='placeholder_map.json', s2_light_yield_map='placeholder_map.json', s1_patterns_file=None, s2_patterns_file=None))) results = [] def finish_results(): nonlocal results records = np.concatenate(results) # In strax data, records are always stored # sorted, baselined and integrated records = strax.sort_by_time(records) strax.baseline(records) strax.integrate(records) results = [] return records for event in mypax.get_events(): event = mypax.process_event(event) pulse_lengths = np.array([p.length for p in event.pulses]) n_records_tot = strax.records_needed(pulse_lengths, samples_per_record).sum() records = np.zeros(n_records_tot, dtype=strax.record_dtype(samples_per_record)) output_record_index = 0 # Record offset in data for p in event.pulses: n_records = strax.records_needed(p.length, samples_per_record) for rec_i in range(n_records): r = records[output_record_index] r['time'] = (event.start_time + p.left * 10 + rec_i * samples_per_record * 10) r['channel'] = p.channel r['pulse_length'] = p.length r['record_i'] = rec_i r['dt'] = 10 # How much are we storing in this record? if rec_i != n_records - 1: # There's more chunks coming, so we store a full chunk n_store = samples_per_record assert p.length > samples_per_record * (rec_i + 1) else: # Just enough to store the rest of the data # Note it's not p.length % samples_per_record!!! # (that would be zero if we have to store a full record) n_store = p.length - samples_per_record * rec_i assert 0 <= n_store <= samples_per_record r['length'] = n_store offset = rec_i * samples_per_record r['data'][:n_store] = p.raw_data[offset:offset + n_store] output_record_index += 1 results.append(records) if len(results) >= events_per_chunk: yield finish_results() mypax.shutdown() if len(results): yield finish_results()