def test_root_output(self):
# Get an event
mypax = core.Processor(config_names='XENON100',
config_dict={
'pax': {
'events_to_process': [0],
'output': 'Dummy.DummyOutput',
'encoder_plugin': None
}
})
mypax.run()
event = mypax.get_plugin_by_name('DummyOutput').last_event
del mypax
# Write same event to ROOT
mypax = core.Processor(config_names='XENON100',
config_dict={
'pax': {
'events_to_process': [0],
'output_name': 'test_root_output'
}
})
mypax.run()
del mypax
self.assertTrue(os.path.exists('test_root_output.root'))
self.assertTrue(hasattr(ROOT, 'Peak'))
# Can't test event class loading, event class already loaded during writing
# ROOTClass.load_pax_event_class_from_root('test_root_output.root')
f = ROOT.TFile('test_root_output.root')
t = f.Get('tree')
t.GetEntry(0)
root_event = t.events
self.assertEqual(len(root_event.peaks), len(event.peaks))
for i in range(len(event.peaks)):
peak = event.peaks[i]
root_peak = root_event.peaks[i]
self.assertEqual(peak.type, root_peak.type)
# 5th or 6th significant figure appears to be different.. float precision difference?
self.assertAlmostEqual(peak.area,
root_peak.area,
delta=0.0001 * max(1, peak.area))
# Check area per channel
self.assertAlmostEqual(peak.area, peak.area_per_channel.sum())
self.assertAlmostEqual(peak.area,
sum(root_peak.area_per_channel),
delta=0.0001 * max(1, peak.area))
np.testing.assert_array_almost_equal(
peak.area_per_channel,
np.array(list(root_peak.area_per_channel)),
decimal=4)
def test_dummy_transform_plugin(self):
mypax = core.Processor(config_dict={
'pax': {
'plugin_group_names': ['bla'],
'bla': 'Dummy.DummyTransform'
}
},
just_testing=True)
self.assertIsInstance(mypax, core.Processor)
self.assertIsInstance(mypax.action_plugins[0], plugin.TransformPlugin)
def test_dummy_input_plugin(self):
mypax = core.Processor(config_dict={
'pax': {
'plugin_group_names': ['input'],
'input': 'Dummy.DummyInput'
}
},
just_testing=True)
self.assertIsInstance(mypax, core.Processor)
self.assertIsInstance(mypax.input_plugin, plugin.InputPlugin)
def setUp(self):
self.pax = core.Processor(config_names='XENON100',
just_testing=True,
config_dict={
'pax': {
'plugin_group_names': ['test'],
'test': 'MaxPMT.PosRecMaxPMT'
}
})
self.plugin = self.pax.get_plugin_by_name('PosRecMaxPMT')
def test_get_input_plugin_by_name(self):
mypax = core.Processor(config_dict={
'pax': {
'plugin_group_names': ['input'],
'input': 'Dummy.DummyInput'
}
},
just_testing=True)
pl = mypax.get_plugin_by_name('DummyInput')
self.assertIsInstance(pl, plugin.InputPlugin)
self.assertEqual(pl.__class__.__name__, 'DummyInput')
def setUp(self):
self.pax = core.Processor(config_names='XENON100',
just_testing=True,
config_dict={
'pax': {
'plugin_group_names': ['test'],
'test':
'HitpatternSpread.HitpatternSpread'
}
})
self.plugin = self.pax.get_plugin_by_name('HitpatternSpread')
def setUp(self): # noqa
self.pax = core.Processor(config_names='XENON100',
just_testing=True,
config_dict={
'pax': {
'plugin_group_names': ['test'],
'test': 'CheckPulses.CheckBoundsAndCount'},
'CheckPulses.CheckBoundsAndCount': {
'truncate_pulses_partially_outside': True}})
self.plugin = self.pax.get_plugin_by_name('CheckBoundsAndCount')
self.baseline = self.pax.config['DEFAULT']['digitizer_reference_baseline']
def test_dummy_output_plugin(self):
mypax = core.Processor(config_dict={
'pax': {
'plugin_group_names': ['output'],
'encoder_plugin': None,
'output': 'Dummy.DummyOutput'
}
},
just_testing=True)
self.assertIsInstance(mypax, core.Processor)
self.assertIsInstance(mypax.action_plugins[0], plugin.OutputPlugin)
def setUp(self):
self.pax = core.Processor(config_names='XENON1T',
just_testing=True,
config_dict={
'pax': {
'plugin_group_names': ['test'],
'look_for_config_in_runs_db': False,
'test': 'NeuralNet.PosRecNeuralNet'
}
})
self.plugin = self.pax.get_plugin_by_name('PosRecNeuralNet')
def setUp(self):
self.pax = core.Processor(
config_names='XENON100',
just_testing=True,
config_dict={
'pax': {
'plugin_group_names': ['test'],
'test': 'CheckPulses.ConcatenateAdjacentPulses'
}
})
self.plugin = self.pax.get_plugin_by_name('ConcatenateAdjacentPulses')
def test_pax_config_string(self):
""" Similar, but using an almost-empty config string """
mypax = core.Processor(config_string="[pax]", just_testing=True)
self.assertIsInstance(mypax, core.Processor)
# Make sure the configuration is actually empty, and a default config did not sneakily get loaded...
# Note plugin_group_names gets autoset during config init
self.assertEqual(mypax.config, {
'DEFAULT': {},
'pax': {
'plugin_group_names': []
}
})
def test_hitfinder(self):
# Integration test for the hitfinder
self.pax = core.Processor(
config_names='XENON100',
just_testing=True,
config_dict={
'pax': {
'plugin_group_names': ['test'],
'encoder_plugin':
None,
'decoder_plugin':
None,
'test':
['PulseProperties.PulseProperties', 'HitFinder.FindHits']
},
'HitFinder.FindHits': {
'left_extension': 0,
'right_extension': 0
}
})
for test_w, hit_bounds, pulse_min, pulse_max in (
# Keep in mind the hitfinder flips the pulse...
[np.zeros(100), [], 0, 0],
[np.ones(100), [], 0, 0],
[-3 * np.ones(100), [], 0, 0],
[self.peak_at(70, amplitude=-100, width=4), [[70, 73]], 0, 100],
[
self.peak_at(70, amplitude=-100, width=4) +
self.peak_at(80, amplitude=10, width=4), [[70, 73]], -10, 100
],
[
self.peak_at(70, amplitude=-100, width=4) +
self.peak_at(80, amplitude=-100, width=4), [[70, 73],
[80, 83]], 0, 100
],
):
e = datastructure.Event(
n_channels=self.pax.config['DEFAULT']['n_channels'],
start_time=0,
sample_duration=self.pax.config['DEFAULT']['sample_duration'],
stop_time=int(1e6),
pulses=[
dict(left=0,
raw_data=np.array(test_w).astype(np.int16),
channel=1)
])
e = self.pax.process_event(e)
self.assertEqual(hit_bounds, [[hit['left'], hit['right']]
for hit in e.all_hits])
self.assertEqual(pulse_min, e.pulses[0].minimum)
self.assertEqual(pulse_max, e.pulses[0].maximum)
delattr(self, 'pax')
def test_process_single_xed_event(self):
""" Process the first event from the XED file.
"""
config = {
'pax': {
'events_to_process': [0],
'encoder_plugin': None,
'output': 'Dummy.DummyOutput'
}
}
mypax = core.Processor(config_names='XENON100', config_dict=config)
mypax.run()
def test_simulator(self):
""" Process the events in dummy_waveforms.csv
"""
mypax = core.Processor(config_names=['XENON100', 'Simulation'],
config_dict={
'pax': {
'encoder_plugin': None,
'output': 'Dummy.DummyOutput'
}
})
mypax.run()
pl = mypax.get_plugin_by_name('DummyOutput')
self.assertIsInstance(pl, plugin.OutputPlugin)
def test_get_events(self):
""" Test getting events from the input plugin
"""
mypax = core.Processor(config_dict={
'pax': {
'plugin_group_names': ['input'],
'input': 'Dummy.DummyInput'
}
},
just_testing=True)
self.assertTrue(inspect.isgeneratorfunction(mypax.get_events))
event_generator = mypax.get_events()
event = next(event_generator)
self.assertIsInstance(event, datastructure.Event)
def test_process_empty_event(self):
""" Test processing without processing plugins defined
"""
mypax = core.Processor(config_dict={
'pax': {
'plugin_group_names': ['input'],
'input': 'Dummy.DummyInput'
}
},
just_testing=True)
event_generator = mypax.get_events()
event = next(event_generator)
event = mypax.process_event(event)
self.assertIsInstance(event, datastructure.Event)
def test_pax_minimal(self):
""" The smallest possible test that actually instantiates the processor.
Does not load any plugins or configuration
"""
mypax = core.Processor(config_dict={'pax': {}}, just_testing=True)
self.assertIsInstance(mypax, core.Processor)
# Make sure the configuration is actually empty, and a default config did not sneakily get loaded...
# Note plugin_group_names gets autoset during config init
self.assertEqual(mypax.config, {
'DEFAULT': {},
'pax': {
'plugin_group_names': []
}
})
def raw_data_processor(input_file_or_directory, config_override=None):
"""Return a raw data processor which reads events from input_file_or_directory
config_override can be used to set additional pax options
"""
if config_override is None:
config_override = {}
# Add the input name to the config_override
# Apply the user overrides, section by section
config_override.setdefault('pax', {})
config_override['pax']['input_name'] = input_file_or_directory
return core.Processor(config_names=hax.config['experiment'],
config_dict=config_override)
def setUp(self): # noqa
self.pax = core.Processor(config_names='XENON100',
just_testing=True,
config_dict={
'pax': {
'plugin_group_names': ['test'],
'test':
'WeightedSum.PosRecWeightedSum'
}
})
self.posrec_plugin = self.pax.get_plugin_by_name('PosRecWeightedSum')
self.e = Event.empty_event()
self.e.peaks.append(Peak({'left': 5, 'right': 9, 'type': 's2'}))
def test_get_plugin_by_name(self):
mypax = core.Processor(config_dict={
'pax': {
'plugin_group_names': ['bla'],
'bla': [
'Dummy.DummyTransform', 'Dummy.DummyTransform2',
'Dummy.DummyTransform3'
]
}
},
just_testing=True)
self.assertIsInstance(mypax, core.Processor)
pl = mypax.get_plugin_by_name('DummyTransform2')
self.assertIsInstance(pl, plugin.TransformPlugin)
self.assertEqual(pl.__class__.__name__, 'DummyTransform2')
def setUp(self): # noqa
self.pax = core.Processor(config_names='XENON100',
just_testing=True,
config_dict={
'pax': {
'plugin_group_names': ['test'],
'test':
'BuildInteractions.BuildInteractions'
},
'BuildInteractions.BuildInteractions': {
'pair_n_s2s': 100,
'pair_n_s1s': 100,
's2_pairing_threshold':
101 * (7 + 1) + 1,
'xy_posrec_preference': ['a', 'b']
}
})
self.plugin = self.pax.get_plugin_by_name('BuildInteractions')
def setUp(self):
self.pax = core.Processor(config_names='XENON100',
just_testing=True,
config_dict={
'pax': {
'plugin_group_names': ['test'],
'test': 'ZLE.SoftwareZLE'
},
'ZLE.SoftwareZLE': {
'zle_threshold': 40,
'samples_to_store_before': 50,
'samples_to_store_after': 50,
'max_intervals': 32,
'special_thresholds': {}
}
})
self.plugin = self.pax.get_plugin_by_name('SoftwareZLE')
def test_process_event_list(self):
""" Take a list of event numbers from a file
"""
with open('temp_eventlist.txt', mode='w') as outfile:
outfile.write("0\n7\n")
config = {
'pax': {
'event_numbers_file': 'temp_eventlist.txt',
'plugin_group_names': ['input', 'output'],
'encoder_plugin': None,
'output': 'Dummy.DummyOutput'
}
}
mypax = core.Processor(config_names='XENON100', config_dict=config)
mypax.run()
self.assertEqual(
mypax.get_plugin_by_name('DummyOutput').last_event.event_number, 7)
os.remove('temp_eventlist.txt')
def test_custom_plugin_location(self):
"""Tests loading a plugin from a custom location"""
tempdir = tempfile.mkdtemp()
with open(os.path.join(tempdir, 'temp_plugin_file.py'),
mode='w') as outfile:
outfile.write(dummy_plugin)
mypax = core.Processor(config_dict={
'pax': {
'plugin_group_names': ['bla'],
'plugin_paths': [tempdir],
'bla': 'temp_plugin_file.FunkyTransform'
}
},
just_testing=True)
self.assertIsInstance(mypax, core.Processor)
pl = mypax.get_plugin_by_name('FunkyTransform')
self.assertIsInstance(pl, plugin.TransformPlugin)
self.assertTrue(hasattr(pl, 'gnork'))
shutil.rmtree(tempdir)
def setUp(self):
self.pax = core.Processor(
config_names='XENON100',
just_testing=True,
config_dict={
'pax': {
'plugin_group_names': ['test'],
'test': [
'WeightedSum.PosRecWeightedSum',
'TopPatternFit.PosRecTopPatternFit'
],
'logging_level':
'debug'
}
})
self.guess_plugin = self.pax.get_plugin_by_name('PosRecWeightedSum')
self.plugin = self.pax.get_plugin_by_name('PosRecTopPatternFit')
self.e = Event.empty_event()
def setUp(self):
self.pax = core.Processor(
config_names='XENON1T',
just_testing=True,
config_dict={
'pax': {
'plugin_group_names': ['test'],
'look_for_config_in_runs_db':
False,
'test': [
'WeightedSum.PosRecWeightedSum',
'TopPatternFit.PosRecTopPatternFunctionFit'
],
'logging_level':
'debug'
}
})
self.guess_plugin = self.pax.get_plugin_by_name('PosRecWeightedSum')
self.plugin = self.pax.get_plugin_by_name(
'PosRecTopPatternFunctionFit')
def test_plotting(self):
""" Plot the first event from the default XED file
"""
import matplotlib
# Force matplotlib to switch to a non-GUI backend, so the test runs on Travis
matplotlib.pyplot.switch_backend('Agg')
mypax = core.Processor(config_names='XENON100',
config_dict={
'pax': {
'output': 'Plotting.PlotEventSummary',
'pre_output': [],
'encoder_plugin': None,
'events_to_process': [0],
'output_name': 'plots_test'
}
})
mypax.run()
self.assertTrue(os.path.exists('./plots_test'))
self.assertTrue(os.path.exists('./plots_test/000000_000000.png'))
shutil.rmtree('plots_test')
def test_process_single_xed_event_olddsp(self):
""" Process the first event from the XED file using Xerawdp matching config
"""
mypax = core.Processor(config_names=['XENON100', 'XerawdpImitation'],
config_dict={
'pax': {
'events_to_process': [0],
'encoder_plugin': None,
'output': 'Dummy.DummyOutput'
}
})
mypax.run()
pl = mypax.get_plugin_by_name('DummyOutput')
self.assertIsInstance(pl, plugin.OutputPlugin)
e = pl.last_event
self.assertIsInstance(e, datastructure.Event)
# Check that the peak areas remain the same
self.assertEqual([x.area for x in e.peaks], [
176279.0866616674, 736.576200034518, 611.0961166092862,
129.12023409842166, 88.43269016354068, 16.19189004866498,
430.28177885354137, 1.9494012301013646, 1.52583418095758,
1.5248293965443862, 1.5214431653719354, 1.1431245035453605,
1.119126521198631, 0.8370846398458212, 0.3965132112382404
])
def pax_to_records(input_filename,
samples_per_record=strax.DEFAULT_RECORD_LENGTH,
events_per_chunk=10):
"""Return pulse records array from pax zip input_filename
This only works if you have pax installed in your strax environment,
which is somewhat tricky.
"""
# Monkeypatch matplotlib so pax is importable
# See https://github.com/XENON1T/pax/pull/734
import matplotlib
matplotlib._cntr = None
from pax import core # Pax is not a dependency
mypax = core.Processor(
'XENON1T',
config_dict=dict(
pax=dict(look_for_config_in_runs_db=False,
plugin_group_names=['input'],
encoder_plugin=None,
input_name=input_filename),
# Fast startup: skip loading big maps
WaveformSimulator=dict(s1_light_yield_map='placeholder_map.json',
s2_light_yield_map='placeholder_map.json',
s1_patterns_file=None,
s2_patterns_file=None)))
print(f"Starting conversion, {events_per_chunk} evt/chunk")
results = []
def finish_results():
nonlocal results
records = np.concatenate(results)
# In strax data, records are always stored
# sorted, baselined and integrated
records = strax.sort_by_time(records)
strax.baseline(records)
strax.integrate(records)
print("Returning %d records" % len(records))
results = []
return records
for event in mypax.get_events():
event = mypax.process_event(event)
if not len(event.pulses):
# Triggerless pax data contains many empty events
# at the end. With the fixed events per chunk setting
# this can lead to empty files, which confuses strax.
continue
pulse_lengths = np.array([p.length for p in event.pulses])
n_records_tot = records_needed(pulse_lengths, samples_per_record).sum()
records = np.zeros(n_records_tot,
dtype=strax.record_dtype(samples_per_record))
output_record_index = 0 # Record offset in data
for p in event.pulses:
n_records = records_needed(p.length, samples_per_record)
for rec_i in range(n_records):
r = records[output_record_index]
r['time'] = (event.start_time + p.left * 10 +
rec_i * samples_per_record * 10)
r['channel'] = p.channel
r['pulse_length'] = p.length
r['record_i'] = rec_i
r['dt'] = 10
# How much are we storing in this record?
if rec_i != n_records - 1:
# There's more chunks coming, so we store a full chunk
n_store = samples_per_record
assert p.length > samples_per_record * (rec_i + 1)
else:
# Just enough to store the rest of the data
# Note it's not p.length % samples_per_record!!!
# (that would be zero if we have to store a full record)
n_store = p.length - samples_per_record * rec_i
assert 0 <= n_store <= samples_per_record
r['length'] = n_store
offset = rec_i * samples_per_record
r['data'][:n_store] = p.raw_data[offset:offset + n_store]
output_record_index += 1
results.append(records)
if len(results) >= events_per_chunk:
yield finish_results()
mypax.shutdown()
if len(results):
y = finish_results()
if len(y):
yield y
def pax_to_records(input_filename,
samples_per_record=strax.DEFAULT_RECORD_LENGTH,
events_per_chunk=10):
"""Return pulse records array from pax zip input_filename"""
from pax import core # Pax is not a dependency
mypax = core.Processor('XENON1T', config_dict=dict(
pax=dict(
look_for_config_in_runs_db=False,
plugin_group_names=['input'],
encoder_plugin=None,
input_name=input_filename),
# Fast startup: skip loading big maps
WaveformSimulator=dict(
s1_light_yield_map='placeholder_map.json',
s2_light_yield_map='placeholder_map.json',
s1_patterns_file=None,
s2_patterns_file=None)))
results = []
def finish_results():
nonlocal results
records = np.concatenate(results)
# In strax data, records are always stored
# sorted, baselined and integrated
records = strax.sort_by_time(records)
strax.baseline(records)
strax.integrate(records)
results = []
return records
for event in mypax.get_events():
event = mypax.process_event(event)
pulse_lengths = np.array([p.length
for p in event.pulses])
n_records_tot = strax.records_needed(pulse_lengths,
samples_per_record).sum()
records = np.zeros(n_records_tot,
dtype=strax.record_dtype(samples_per_record))
output_record_index = 0 # Record offset in data
for p in event.pulses:
n_records = strax.records_needed(p.length, samples_per_record)
for rec_i in range(n_records):
r = records[output_record_index]
r['time'] = (event.start_time
+ p.left * 10
+ rec_i * samples_per_record * 10)
r['channel'] = p.channel
r['pulse_length'] = p.length
r['record_i'] = rec_i
r['dt'] = 10
# How much are we storing in this record?
if rec_i != n_records - 1:
# There's more chunks coming, so we store a full chunk
n_store = samples_per_record
assert p.length > samples_per_record * (rec_i + 1)
else:
# Just enough to store the rest of the data
# Note it's not p.length % samples_per_record!!!
# (that would be zero if we have to store a full record)
n_store = p.length - samples_per_record * rec_i
assert 0 <= n_store <= samples_per_record
r['length'] = n_store
offset = rec_i * samples_per_record
r['data'][:n_store] = p.raw_data[offset:offset + n_store]
output_record_index += 1
results.append(records)
if len(results) >= events_per_chunk:
yield finish_results()
mypax.shutdown()
if len(results):
yield finish_results()
