def compute(self, raw_records):
# Convert everything to the records data type -- adds extra fields.
records = strax.raw_to_records(raw_records)
del raw_records
# calculate baseline and baseline rms
strax.baseline(records,
baseline_samples=self.config['baseline_samples'],
flip=True)
# find all hits
hits = strax.find_hits(
records,
min_amplitude=self.hit_thresholds,
min_height_over_noise=self.config['hit_min_height_over_noise'],
)
# sort hits by record_i and time, then find LED hit and afterpulse
# hits within the same record
hits_ap = find_ap(
hits,
records,
LED_window_left=self.config['LED_window_left'],
LED_window_right=self.config['LED_window_right'],
hit_left_extension=self.hit_left_extension,
hit_right_extension=self.hit_right_extension,
)
hits_ap['area_pe'] = hits_ap['area'] * self.to_pe[hits_ap['channel']]
hits_ap['height_pe'] = hits_ap['height'] * self.to_pe[
hits_ap['channel']]
return hits_ap
def test_lone_hits_integration_bounds(records, left_extension, right_extension):
"""
Loops over hits and tests if integration bounds overlap.
"""
n_channel = 0
if len(records):
n_channel = records['channel'].max()+1
hits = strax.find_hits(records, np.ones(n_channel))
strax.find_hit_integration_bounds(hits,
np.zeros(0, dtype=strax.time_dt_fields),
records,
(left_extension, right_extension),
n_channel,
allow_bounds_beyond_records=False
)
_test_overlap(hits)
hits['left_integration'] = 0
hits['right_integration'] = 0
strax.find_hit_integration_bounds(hits,
np.zeros(0, dtype=strax.time_dt_fields),
records,
(left_extension, right_extension),
n_channel,
allow_bounds_beyond_records=True
)
_test_overlap(hits)
def compute(self, records_nv, start, end):
hits = strax.find_hits(records_nv, min_amplitude=self.config['hit_min_amplitude_nv'])
hits = remove_switched_off_channels(hits, self.to_pe)
temp_hitlets = strax.create_hitlets_from_hits(hits,
self.config['save_outside_hits_nv'],
self.channel_range,
chunk_start=start,
chunk_end=end)
del hits
# Get hitlet data and split hitlets:
temp_hitlets = strax.get_hitlets_data(temp_hitlets, records_nv, to_pe=self.to_pe)
temp_hitlets = strax.split_peaks(temp_hitlets,
records_nv,
self.to_pe,
data_type='hitlets',
algorithm='local_minimum',
min_height=self.config['min_split_nv'],
min_ratio=self.config['min_split_ratio_nv']
)
# Compute other hitlet properties:
# We have to loop here 3 times over all hitlets...
strax.hitlet_properties(temp_hitlets)
entropy = strax.conditional_entropy(temp_hitlets, template='flat', square_data=False)
temp_hitlets['entropy'][:] = entropy
# Remove data field:
hitlets = np.zeros(len(temp_hitlets), dtype=strax.hitlet_dtype())
strax.copy_to_buffer(temp_hitlets, hitlets, '_copy_hitlets')
return hitlets
def compute(self, records):
r = records
hits = strax.find_hits(r, threshold=self.config['hit_threshold'])
hits = strax.sort_by_time(hits)
peaks = strax.find_peaks(
hits,
self.config['to_pe'],
result_dtype=self.dtype,
gap_threshold=self.config['peak_gap_threshold'],
left_extension=self.config['peak_left_extension'],
right_extension=self.config['peak_right_extension'],
min_channels=self.config['peak_min_chan'],
min_area=self.config['peak_min_area'],
max_duration=self.config['peak_max_duration'])
strax.sum_waveform(peaks, r, adc_to_pe=self.config['to_pe'])
peaks = peaks[peaks['dt'] > 0] # removes strange edge case
peaks = strax.split_peaks(peaks,
r,
self.config['to_pe'],
min_height=self.config['split_min_height'],
min_ratio=self.config['split_min_ratio'])
strax.compute_widths(peaks)
return peaks
def compute(self, raw_records):
# Remove records from channels for which the gain is unknown
r = raw_records[raw_records['channel'] < len(to_pe)]
hits = strax.find_hits(r)
strax.cut_outside_hits(r, hits)
return r
def compute(self, raw_records):
# Remove records from funny channels (if present)
r = raw_records[raw_records['channel'] < len(to_pe)]
# Do not trust in DAQ + strax.baseline to leave the
# out-of-bounds samples to zero.
strax.zero_out_of_bounds(r)
if self.config['s2_tail_veto']:
# Experimental data reduction
r = strax.exclude_tails(r, to_pe)
# Find hits before filtering
hits = strax.find_hits(r)
if self.config['filter']:
# Filter to concentrate the PMT pulses
strax.filter_records(
r, np.array(self.config['filter']))
le, re = self.config['save_outside_hits']
r = strax.cut_outside_hits(r, hits,
left_extension=le,
right_extension=re)
# Probably overkill, but just to be sure...
strax.zero_out_of_bounds(r)
return r
def compute(self, raw_records_coin_nv):
# Do not trust in DAQ + strax.baseline to leave the
# out-of-bounds samples to zero.
r = strax.raw_to_records(raw_records_coin_nv)
del raw_records_coin_nv
r = strax.sort_by_time(r)
strax.zero_out_of_bounds(r)
strax.baseline(r,
baseline_samples=self.baseline_samples,
flip=True)
if self.config['min_samples_alt_baseline_nv']:
m = r['pulse_length'] > self.config['min_samples_alt_baseline_nv']
if np.any(m):
# Correcting baseline after PMT saturated signals
r[m] = median_baseline(r[m])
strax.integrate(r)
strax.zero_out_of_bounds(r)
hits = strax.find_hits(r, min_amplitude=self.hit_thresholds)
le, re = self.config['save_outside_hits_nv']
r = strax.cut_outside_hits(r, hits, left_extension=le, right_extension=re)
strax.zero_out_of_bounds(r)
return r
def compute(self, raw_records_coin_nv):
# Do not trust in DAQ + strax.baseline to leave the
# out-of-bounds samples to zero.
r = strax.raw_to_records(raw_records_coin_nv)
del raw_records_coin_nv
r = strax.sort_by_time(r)
strax.zero_out_of_bounds(r)
strax.baseline(r,
baseline_samples=self.config['baseline_samples_nv'],
flip=True)
strax.integrate(r)
strax.zero_out_of_bounds(r)
hits = strax.find_hits(
r, min_amplitude=self.config['hit_min_amplitude_nv'])
le, re = self.config['save_outside_hits_nv']
r = strax.cut_outside_hits(r,
hits,
left_extension=le,
right_extension=re)
strax.zero_out_of_bounds(r)
rlinks = strax.record_links(r)
r = clean_up_empty_records(r, rlinks, only_last=True)
return r
def _find_hits(r):
hits = strax.find_hits(r, threshold=0)
# Test pulses have dt=1 and time=0
# TODO: hm, maybe this doesn't test everything
np.testing.assert_equal(hits['time'], hits['left'])
# NB: exclusive right bound, no + 1 here
np.testing.assert_equal(hits['length'], hits['right'] - hits['left'])
return list(zip(hits['left'], hits['right']))
def test__build_hit_waveform(records):
hits = strax.find_hits(records, np.ones(10000))
for h in hits:
hit_waveform = np.zeros(len(records[0]['data']))
_ = strax.processing.peak_building._build_hit_waveform(
h, records[h['record_i']], hit_waveform)
assert h['area'] == np.sum(hit_waveform), 'Got worng area!'
def compute(self, raw_records):
# Do not trust in DAQ + strax.baseline to leave the
# out-of-bounds samples to zero.
strax.zero_out_of_bounds(raw_records)
##
# Split off non-TPC records and count TPC pulses
# (perhaps we should migrate this to DAQRreader in the future)
##
r, other = channel_split(raw_records, n_tpc)
pulse_counts = count_pulses(r, n_tpc)
diagnostic_records, aqmon_records = channel_split(other, 254)
##
# Process the TPC records
##
if self.config['tail_veto_threshold'] and len(r):
r, r_vetoed, veto_regions = software_he_veto(
r,
self.to_pe,
area_threshold=self.config['tail_veto_threshold'],
veto_length=self.config['tail_veto_duration'],
veto_res=self.config['tail_veto_resolution'],
pass_veto_fraction=self.config['tail_veto_pass_fraction'],
pass_veto_extend=self.config['tail_veto_pass_extend'])
# In the future, we'll probably want to sum the waveforms
# inside the vetoed regions, so we can still save the "peaks".
del r_vetoed
else:
veto_regions = np.zeros(0, dtype=strax.hit_dtype)
if len(r):
# Find hits
# -- before filtering,since this messes with the with the S/N
hits = strax.find_hits(r, threshold=self.config['hit_threshold'])
if self.config['pmt_pulse_filter']:
# Filter to concentrate the PMT pulses
strax.filter_records(r,
np.array(self.config['pmt_pulse_filter']))
le, re = self.config['save_outside_hits']
r = strax.cut_outside_hits(r,
hits,
left_extension=le,
right_extension=re)
# Probably overkill, but just to be sure...
strax.zero_out_of_bounds(r)
return dict(records=r,
diagnostic_records=diagnostic_records,
aqmon_records=aqmon_records,
pulse_counts=pulse_counts,
veto_regions=veto_regions)
def compute(self, raw_records_aqmon):
rec = strax.raw_to_records(raw_records_aqmon)
strax.sort_by_time(rec)
strax.zero_out_of_bounds(rec)
strax.baseline(rec,
baseline_samples=self.config['baseline_samples_aqmon'],
flip=True)
aqmon_hits = strax.find_hits(
rec, min_amplitude=self.config['hit_min_amplitude_aqmon'])
return aqmon_hits
def _find_hits(r):
hits = strax.find_hits(r, threshold=0)
# Test pulses have dt=1 and time=0
# TODO: hm, maybe this doesn't test everything
np.testing.assert_equal(hits['time'], hits['left'])
# NB: exclusive right bound, no + 1 here
np.testing.assert_equal(hits['length'], hits['right'] - hits['left'])
for h in hits:
q = r[h['record_i']]
assert q['data'][h['left']:h['right']].sum() == h['area']
return list(zip(hits['left'], hits['right']))
def compute(self, records_nv, start, end):
# Search again for hits in records:
hits = strax.find_hits(
records_nv, min_amplitude=self.config['hit_min_amplitude_nv'])
# Merge concatenate overlapping within a channel. This is important
# in case hits were split by record boundaries. In case we
# accidentally concatenate two PMT signals we split them later again.
hits = strax.concat_overlapping_hits(
hits, self.config['save_outside_hits_nv'],
self.config['channel_map']['nveto'], start, end)
hits = strax.sort_by_time(hits)
# Now convert hits into temp_hitlets including the data field:
if len(hits):
nsamples = hits['length'].max()
else:
nsamples = 0
temp_hitlets = np.zeros(
len(hits), strax.hitlet_with_data_dtype(n_samples=nsamples))
# Generating hitlets and copying relevant information from hits to hitlets.
# These hitlets are not stored in the end since this array also contains a data
# field which we will drop later.
strax.refresh_hit_to_hitlets(hits, temp_hitlets)
del hits
# Get hitlet data and split hitlets:
strax.get_hitlets_data(temp_hitlets, records_nv, to_pe=self.to_pe)
temp_hitlets = strax.split_peaks(
temp_hitlets,
records_nv,
self.to_pe,
data_type='hitlets',
algorithm='local_minimum',
min_height=self.config['min_split_nv'],
min_ratio=self.config['min_split_ratio_nv'])
# Compute other hitlet properties:
# We have to loop here 3 times over all hitlets...
strax.hitlet_properties(temp_hitlets)
entropy = strax.conditional_entropy(temp_hitlets,
template='flat',
square_data=False)
temp_hitlets['entropy'][:] = entropy
strax.compute_widths(temp_hitlets)
# Remove data field:
hitlets = np.zeros(len(temp_hitlets), dtype=strax.hitlet_dtype())
drop_data_field(temp_hitlets, hitlets)
return hitlets
def compute(self, raw_records):
# Remove records from channels for which the gain is unknown
r = raw_records[raw_records['channel'] < len(to_pe)]
# Experimental data reduction: disabled
# Seems to remove many S2s since it triggers on S1s!
# (perhaps due to larger amount of afterpuless
#r = strax.exclude_tails(r, to_pe)
hits = strax.find_hits(r)
strax.cut_outside_hits(r, hits)
return r
def test_splitter_outer():
data = [0, 2, 2, 0, 2, 2, 1]
records = np.zeros(1, dtype=strax.record_dtype(len(data)))
records['dt'] = 1
records['data'] = data
records['length'] = len(data)
records['pulse_length'] = len(data)
to_pe = np.ones(10)
hits = strax.find_hits(records, np.ones(1))
hits['left_integration'] = hits['left']
hits['right_integration'] = hits['right']
peaks = np.zeros(1, dtype=strax.peak_dtype())
hitlets = np.zeros(1, dtype=strax.hitlet_with_data_dtype(10))
for data_type in (peaks, hitlets):
data_type['dt'] = 1
data_type['data'][0, :len(data)] = data
data_type['length'] = len(data)
rlinks = strax.record_links(records)
peaks = strax.split_peaks(peaks,
hits,
records,
rlinks,
to_pe,
algorithm='local_minimum',
data_type='peaks',
min_height=1,
min_ratio=0)
hitlets = strax.split_peaks(hitlets,
hits,
records,
rlinks,
to_pe,
algorithm='local_minimum',
data_type='hitlets',
min_height=1,
min_ratio=0)
for name, data_type in zip(('peaks', 'hitlets'), (peaks, hitlets)):
data = data_type[0]['data'][:data_type[0]['length']]
assert np.all(
data == [0, 2, 2]
), f'Wrong split for {name}, got {data}, expected {[0, 2, 2]}.'
data = data_type[1]['data'][:data_type[1]['length']]
assert np.all(
data == [0, 2, 2, 1]
), f'Wrong split for {name}, got {data}, expected {[0, 2, 2, 1]}.'
def compute(self, records, peaks):
r = records
p = peaks
hits = strax.find_hits(r)
hits = strax.sort_by_time(hits)
hit_mean_times = hits['time'] + (hits['length'] / 2.0
) # hit "mean" time
peak_mean_times = p['time'] + (p['length'] / 2.0) # peak "mean" time
tight_coin = self.get_tight_coin(
hit_mean_times, peak_mean_times,
self.config['tight_coincidence_window_left'],
self.config['tight_coincidence_window_right'])
return dict(tight_coincidence=tight_coin)
def find_aqmon_hits_per_channel(self, records):
"""Allow different thresholds to be applied to different channels"""
aqmon_thresholds = np.zeros(np.max(self.aqmon_channels) + 1)
for hit_threshold, channels in self.hit_min_amplitude_aqmon:
aqmon_thresholds[np.array(channels)] = hit_threshold
# Split the artificial deadtime ones and do those separately if there are any
is_artificial = records['channel'] == AqmonChannels.ARTIFICIAL_DEADTIME
aqmon_hits = strax.find_hits(records[~is_artificial],
min_amplitude=aqmon_thresholds)
if np.sum(is_artificial):
aqmon_hits = np.concatenate(
[aqmon_hits,
self.get_deadtime_hits(records[is_artificial])])
return aqmon_hits
def compute(self, raw_records):
# Remove records from channels for which the gain is unknown
# or low
channels_to_cut = np.argwhere(
self.config['to_pe'] > (adc_to_e / self.config['min_gain']))
r = raw_records
for ch in channels_to_cut.reshape(-1):
r = r[r['channel'] != ch]
strax.zero_out_of_bounds(r)
hits = strax.find_hits(r, threshold=self.config['hit_threshold'])
strax.cut_outside_hits(
r,
hits,
left_extension=self.config['left_cut_extension'],
right_extension=self.config['right_cut_extension'])
return r
def _find_hits(r):
# Test pulses have dt=1 and time=0
# TODO: hm, maybe this doesn't test everything
hits = strax.find_hits(r, min_amplitude=1)
# dt = 1, so:
np.testing.assert_equal(hits['time'], hits['left'])
# NB: exclusive right bound, no + 1 here
np.testing.assert_equal(hits['length'], hits['right'] - hits['left'])
# Check hits are properly integrated
for h in hits:
q = r[h['record_i']]
assert q['data'][h['left']:h['right']].sum() == h['area']
return list(zip(hits['left'], hits['right']))
def test_get_hitlets_data():
dummy_records = [ # Contains Hitlet #:
[
[1, 3, 2, 1, 0, 0],
], # 0
[
[0, 0, 0, 0, 1, 3], # 1
[2, 1, 0, 0, 0, 0]
], #
[
[0, 0, 0, 0, 1, 3], # 2
[2, 1, 0, 1, 3, 2],
], # 3
[
[0, 0, 0, 0, 1, 2], # 4
[2, 2, 2, 2, 2, 2],
[2, 1, 0, 0, 0, 0]
],
[[2, 1, 0, 1, 3, 2]], # 5, 6
[[2, 2, 2, 2, 2, 2]] # 7
]
# Defining the true parameters of the hitlets:
true_area = [7, 7, 7, 6, 18, 3, 6, 12]
true_time = [10, 28, 46, 51, 68, 88, 91, 104]
true_waveform = [[1, 3, 2, 1], [1, 3, 2, 1], [1, 3, 2, 1], [1, 3, 2],
[1, 2, 2, 2, 2, 2, 2, 2, 2, 1], [2, 1], [1, 3, 2],
[2, 2, 2, 2, 2, 2]]
records = _make_fake_records(dummy_records)
hits = strax.find_hits(records, min_amplitude=2)
hits = strax.concat_overlapping_hits(hits, (1, 1), (0, 1), 0, float('inf'))
hitlets = np.zeros(
len(hits),
strax.hitlet_with_data_dtype(n_samples=np.max(hits['length'])))
strax.refresh_hit_to_hitlets(hits, hitlets)
strax.get_hitlets_data(hitlets, records, np.array([1, 1]))
for i, (a, wf, t) in enumerate(zip(true_area, true_waveform, true_time)):
h = hitlets[i]
assert h['area'] == a, f'Hitlet {i} has the wrong area'
assert np.all(h['data'][:h['length']] ==
wf), f'Hitlet {i} has the wrong waveform'
assert h['time'] == t, f'Hitlet {i} has the wrong starttime'
def compute(self, records):
r = records
hits = strax.find_hits(r) # TODO: Duplicate work
hits = strax.sort_by_time(hits)
peaks = strax.find_peaks(hits, to_pe, result_dtype=self.dtype)
strax.sum_waveform(peaks, r, to_pe)
peaks = strax.split_peaks(peaks, r, to_pe)
strax.compute_widths(peaks)
if self.config['diagnose_sorting']:
assert np.diff(r['time']).min() >= 0, "Records not sorted"
assert np.diff(hits['time']).min() >= 0, "Hits not sorted"
assert np.all(peaks['time'][1:] >= strax.endtime(peaks)[:-1]
), "Peaks not disjoint"
return peaks
def test_sum_waveform(records):
# Make a single big peak to contain all the records
n_ch = 100
rlinks = strax.record_links(records)
hits = strax.find_hits(records, np.ones(n_ch))
hits['left_integration'] = hits['left']
hits['right_integration'] = hits['right']
hits = strax.sort_by_time(hits)
peaks = strax.find_peaks(hits,
np.ones(n_ch),
gap_threshold=6,
left_extension=2,
right_extension=3,
min_area=0,
min_channels=1,
max_duration=10_000_000)
strax.sum_waveform(peaks, hits, records, rlinks, np.ones(n_ch))
for p in peaks:
# Area measures must be consistent
area = p['area']
assert area >= 0
assert p['data'].sum() == area
assert p['area_per_channel'].sum() == area
sum_wv = np.zeros(p['length'], dtype=np.float32)
for r in records:
(rs, re), (ps, pe) = strax.overlap_indices(r['time'], r['length'],
p['time'], p['length'])
sum_wv[ps:pe] += r['data'][rs:re]
assert np.all(p['data'][:p['length']] == sum_wv)
# Finally check that we also can use a selection of peaks to sum
strax.sum_waveform(peaks,
hits,
records,
rlinks,
np.ones(n_ch),
select_peaks_indices=np.array([0]))
def test_ext_timings_nv(records):
"""
Little test for nVetoExtTimings.
"""
if not straxen.utilix_is_configured():
return
# several fake records do not have any pulse length
# and channel start at 0, convert to nv:
records['pulse_length'] = records['length']
records['channel'] += 2000
st = straxen.contexts.xenonnt_led()
plugin = st.get_single_plugin('1', 'ext_timings_nv')
hits = strax.find_hits(records, min_amplitude=1)
hitlets = np.zeros(len(hits), strax.hitlet_dtype())
strax.copy_to_buffer(hits, hitlets, '_refresh_hit_to_hitlets')
result = plugin.compute(hitlets, records)
assert len(result) == len(hits)
assert np.all(result['time'] == hits['time'])
assert np.all(result['pulse_i'] == hits['record_i'])
true_dt = hits['time'] - records[hits['record_i']]['time']
assert np.all(result['delta_time'] == true_dt)
def compute(self, records):
r = records
hits = strax.find_hits(r)
# Remove hits in zero-gain channels
# they should not affect the clustering!
hits = hits[self.to_pe[hits['channel']] != 0]
hits = strax.sort_by_time(hits)
peaks = strax.find_peaks(
hits,
self.to_pe,
gap_threshold=self.config['peak_gap_threshold'],
left_extension=self.config['peak_left_extension'],
right_extension=self.config['peak_right_extension'],
min_channels=self.config['peak_min_pmts'],
result_dtype=self.dtype)
strax.sum_waveform(peaks, r, self.to_pe)
peaks = strax.split_peaks(
peaks,
r,
self.to_pe,
min_height=self.config['peak_split_min_height'],
min_ratio=self.config['peak_split_min_ratio'])
strax.compute_widths(peaks)
if self.config['diagnose_sorting']:
assert np.diff(r['time']).min() >= 0, "Records not sorted"
assert np.diff(hits['time']).min() >= 0, "Hits not sorted"
assert np.all(peaks['time'][1:] >= strax.endtime(peaks)[:-1]
), "Peaks not disjoint"
return peaks
def compute(self, raw_records_coin_nv):
# Do not trust in DAQ + strax.baseline to leave the
# out-of-bounds samples to zero.
r = strax.raw_to_records(raw_records_coin_nv)
del raw_records_coin_nv
r = strax.sort_by_time(r)
strax.zero_out_of_bounds(r)
strax.baseline(r, baseline_samples=self.baseline_samples, flip=True)
strax.integrate(r)
strax.zero_out_of_bounds(r)
hits = strax.find_hits(r, min_amplitude=self.hit_thresholds)
le, re = self.config['save_outside_hits_nv']
r = strax.cut_outside_hits(r,
hits,
left_extension=le,
right_extension=re)
strax.zero_out_of_bounds(r)
return r
def software_he_veto(records,
to_pe,
chunk_end,
area_threshold=int(1e5),
veto_length=int(3e6),
veto_res=int(1e3),
pass_veto_fraction=0.01,
pass_veto_extend=3,
max_veto_value=None):
"""Veto veto_length (time in ns) after peaks larger than
area_threshold (in PE).
Further large peaks inside the veto regions are still passed:
We sum the waveform inside the veto region (with time resolution
veto_res in ns) and pass regions within pass_veto_extend samples
of samples with amplitude above pass_veto_fraction times the maximum.
:returns: (preserved records, vetoed records, veto intervals).
:param records: PMT records
:param to_pe: ADC to PE conversion factors for the channels in records.
:param chunk_end: Endtime of chunk to set as maximum ceiling for the veto period
:param area_threshold: Minimum peak area to trigger the veto.
Note we use a much rougher clustering than in later processing.
:param veto_length: Time in ns to veto after the peak
:param veto_res: Resolution of the sum waveform inside the veto region.
Do not make too large without increasing integer type in some strax
dtypes...
:param pass_veto_fraction: fraction of maximum sum waveform amplitude to
trigger veto passing of further peaks
:param pass_veto_extend: samples to extend (left and right) the pass veto
regions.
:param max_veto_value: if not None, pass peaks that exceed this area
no matter what.
"""
veto_res = int(veto_res)
if veto_res > np.iinfo(np.int16).max:
raise ValueError("Veto resolution does not fit 16-bit int")
veto_length = np.ceil(veto_length / veto_res).astype(np.int) * veto_res
veto_n = int(veto_length / veto_res) + 1
# 1. Find large peaks in the data.
# This will actually return big agglomerations of peaks and their tails
peaks = strax.find_peaks(records,
to_pe,
gap_threshold=1,
left_extension=0,
right_extension=0,
min_channels=100,
min_area=area_threshold,
result_dtype=strax.peak_dtype(
n_channels=len(to_pe),
n_sum_wv_samples=veto_n))
# 2a. Set 'candidate regions' at these peaks. These should:
# - Have a fixed maximum length (else we can't use the strax hitfinder on them)
# - Never extend beyond the current chunk
# - Do not overlap
veto_start = peaks['time']
veto_end = np.clip(peaks['time'] + veto_length, None, chunk_end)
veto_end[:-1] = np.clip(veto_end[:-1], None, veto_start[1:])
# 2b. Convert these into strax record-like objects
# Note the waveform is float32 though (it's a summed waveform)
regions = np.zeros(len(veto_start),
dtype=strax.interval_dtype + [
("data", (np.float32, veto_n)),
("baseline", np.float32),
("baseline_rms", np.float32),
("reduction_level", np.int64),
("record_i", np.int64),
("pulse_length", np.int64),
])
regions['time'] = veto_start
regions['length'] = (veto_end - veto_start) // veto_n
regions['pulse_length'] = veto_n
regions['dt'] = veto_res
if not len(regions):
# No veto anywhere in this data
return records, records[:0], np.zeros(0, strax.hit_dtype)
# 3. Find pass_veto regios with big peaks inside the veto regions.
# For this we compute a rough sum waveform (at low resolution,
# without looping over the pulse data)
rough_sum(regions, records, to_pe, veto_n, veto_res)
if max_veto_value is not None:
pass_veto = strax.find_hits(regions, min_amplitude=max_veto_value)
else:
regions['data'] /= np.max(regions['data'], axis=1)[:, np.newaxis]
pass_veto = strax.find_hits(regions, min_amplitude=pass_veto_fraction)
# 4. Extend these by a few samples and inverse to find veto regions
regions['data'] = 1
regions = strax.cut_outside_hits(regions,
pass_veto,
left_extension=pass_veto_extend,
right_extension=pass_veto_extend)
regions['data'] = 1 - regions['data']
veto = strax.find_hits(regions, min_amplitude=1)
# Do not remove very tiny regions
veto = veto[veto['length'] > 2 * pass_veto_extend]
# 5. Apply the veto and return results
veto_mask = strax.fully_contained_in(records, veto) == -1
return tuple(list(mask_and_not(records, veto_mask)) + [veto])
def compute(self, raw_records, start, end):
if self.config['check_raw_record_overlaps']:
check_overlaps(raw_records, n_channels=3000)
# Throw away any non-TPC records; this should only happen for XENON1T
# converted data
raw_records = raw_records[
raw_records['channel'] < self.config['n_tpc_pmts']]
# Convert everything to the records data type -- adds extra fields.
r = strax.raw_to_records(raw_records)
del raw_records
# Do not trust in DAQ + strax.baseline to leave the
# out-of-bounds samples to zero.
# TODO: better to throw an error if something is nonzero
strax.zero_out_of_bounds(r)
strax.baseline(
r,
baseline_samples=self.config['baseline_samples'],
allow_sloppy_chunking=self.config['allow_sloppy_chunking'],
flip=True)
strax.integrate(r)
pulse_counts = count_pulses(r, self.config['n_tpc_pmts'])
pulse_counts['time'] = start
pulse_counts['endtime'] = end
if len(r) and self.hev_enabled:
r, r_vetoed, veto_regions = software_he_veto(
r,
self.to_pe,
end,
area_threshold=self.config['tail_veto_threshold'],
veto_length=self.config['tail_veto_duration'],
veto_res=self.config['tail_veto_resolution'],
pass_veto_extend=self.config['tail_veto_pass_extend'],
pass_veto_fraction=self.config['tail_veto_pass_fraction'],
max_veto_value=self.config['max_veto_value'])
# In the future, we'll probably want to sum the waveforms
# inside the vetoed regions, so we can still save the "peaks".
del r_vetoed
else:
veto_regions = np.zeros(0, dtype=strax.hit_dtype)
if len(r):
# Find hits
# -- before filtering,since this messes with the with the S/N
hits = strax.find_hits(r,
min_amplitude=straxen.hit_min_amplitude(
self.config['hit_min_amplitude']))
if self.config['pmt_pulse_filter']:
# Filter to concentrate the PMT pulses
strax.filter_records(r,
np.array(self.config['pmt_pulse_filter']))
le, re = self.config['save_outside_hits']
r = strax.cut_outside_hits(r,
hits,
left_extension=le,
right_extension=re)
# Probably overkill, but just to be sure...
strax.zero_out_of_bounds(r)
return dict(records=r,
pulse_counts=pulse_counts,
veto_regions=veto_regions)
def compute(self, records, start, end):
r = records
hits = strax.find_hits(r, min_amplitude=self.hit_thresholds)
# Remove hits in zero-gain channels
# they should not affect the clustering!
hits = hits[self.to_pe[hits['channel']] != 0]
hits = strax.sort_by_time(hits)
# Use peaklet gap threshold for initial clustering
# based on gaps between hits
peaklets = strax.find_peaks(
hits,
self.to_pe,
gap_threshold=self.config['peaklet_gap_threshold'],
left_extension=self.config['peak_left_extension'],
right_extension=self.config['peak_right_extension'],
min_channels=self.config['peak_min_pmts'],
result_dtype=self.dtype_for('peaklets'),
max_duration=self.config['peaklet_max_duration'],
)
# Make sure peaklets don't extend out of the chunk boundary
# This should be very rare in normal data due to the ADC pretrigger
# window.
self.clip_peaklet_times(peaklets, start, end)
# Get hits outside peaklets, and store them separately.
# fully_contained is OK provided gap_threshold > extension,
# which is asserted inside strax.find_peaks.
is_lone_hit = strax.fully_contained_in(hits, peaklets) == -1
lone_hits = hits[is_lone_hit]
strax.integrate_lone_hits(
lone_hits,
records,
peaklets,
save_outside_hits=(self.config['peak_left_extension'],
self.config['peak_right_extension']),
n_channels=len(self.to_pe))
# Compute basic peak properties -- needed before natural breaks
hits = hits[~is_lone_hit]
# Define regions outside of peaks such that _find_hit_integration_bounds
# is not extended beyond a peak.
outside_peaks = self.create_outside_peaks_region(peaklets, start, end)
strax.find_hit_integration_bounds(
hits,
outside_peaks,
records,
save_outside_hits=(self.config['peak_left_extension'],
self.config['peak_right_extension']),
n_channels=len(self.to_pe),
allow_bounds_beyond_records=True,
)
# Transform hits to hitlets for naming conventions. A hit refers
# to the central part above threshold a hitlet to the entire signal
# including the left and right extension.
# (We are not going to use the actual hitlet data_type here.)
hitlets = hits
del hits
hitlet_time_shift = (hitlets['left'] -
hitlets['left_integration']) * hitlets['dt']
hitlets['time'] = hitlets['time'] - hitlet_time_shift
hitlets['length'] = (hitlets['right_integration'] -
hitlets['left_integration'])
hitlets = strax.sort_by_time(hitlets)
rlinks = strax.record_links(records)
strax.sum_waveform(peaklets, hitlets, r, rlinks, self.to_pe)
strax.compute_widths(peaklets)
# Split peaks using low-split natural breaks;
# see https://github.com/XENONnT/straxen/pull/45
# and https://github.com/AxFoundation/strax/pull/225
peaklets = strax.split_peaks(
peaklets,
hitlets,
r,
rlinks,
self.to_pe,
algorithm='natural_breaks',
threshold=self.natural_breaks_threshold,
split_low=True,
filter_wing_width=self.config['peak_split_filter_wing_width'],
min_area=self.config['peak_split_min_area'],
do_iterations=self.config['peak_split_iterations'])
# Saturation correction using non-saturated channels
# similar method used in pax
# see https://github.com/XENON1T/pax/pull/712
# Cases when records is not writeable for unclear reason
# only see this when loading 1T test data
# more details on https://numpy.org/doc/stable/reference/generated/numpy.ndarray.flags.html
if not r['data'].flags.writeable:
r = r.copy()
if self.config['saturation_correction_on']:
peak_list = peak_saturation_correction(
r,
rlinks,
peaklets,
hitlets,
self.to_pe,
reference_length=self.config['saturation_reference_length'],
min_reference_length=self.
config['saturation_min_reference_length'])
# Compute the width again for corrected peaks
strax.compute_widths(peaklets, select_peaks_indices=peak_list)
# Compute tight coincidence level.
# Making this a separate plugin would
# (a) doing hitfinding yet again (or storing hits)
# (b) increase strax memory usage / max_messages,
# possibly due to its currently primitive scheduling.
hit_max_times = np.sort(
hitlets['time'] +
hitlets['dt'] * hit_max_sample(records, hitlets) +
hitlet_time_shift # add time shift again to get correct maximum
)
peaklet_max_times = (
peaklets['time'] +
np.argmax(peaklets['data'], axis=1) * peaklets['dt'])
tight_coincidence_channel = get_tight_coin(
hit_max_times, hitlets['channel'], peaklet_max_times,
self.config['tight_coincidence_window_left'],
self.config['tight_coincidence_window_right'], self.channel_range)
peaklets['tight_coincidence'] = tight_coincidence_channel
if self.config['diagnose_sorting'] and len(r):
assert np.diff(r['time']).min(initial=1) >= 0, "Records not sorted"
assert np.diff(
hitlets['time']).min(initial=1) >= 0, "Hits/Hitlets not sorted"
assert np.all(peaklets['time'][1:] >= strax.endtime(peaklets)[:-1]
), "Peaks not disjoint"
# Update nhits of peaklets:
counts = strax.touching_windows(hitlets, peaklets)
counts = np.diff(counts, axis=1).flatten()
peaklets['n_hits'] = counts
return dict(peaklets=peaklets, lone_hits=lone_hits)
def compute(self, records, start, end):
r = records
hits = strax.find_hits(r,
min_amplitude=straxen.hit_min_amplitude(
self.config['hit_min_amplitude']))
# Remove hits in zero-gain channels
# they should not affect the clustering!
hits = hits[self.to_pe[hits['channel']] != 0]
hits = strax.sort_by_time(hits)
# Use peaklet gap threshold for initial clustering
# based on gaps between hits
peaklets = strax.find_peaks(
hits,
self.to_pe,
gap_threshold=self.config['peaklet_gap_threshold'],
left_extension=self.config['peak_left_extension'],
right_extension=self.config['peak_right_extension'],
min_channels=self.config['peak_min_pmts'],
result_dtype=self.dtype_for('peaklets'))
# Make sure peaklets don't extend out of the chunk boundary
# This should be very rare in normal data due to the ADC pretrigger
# window.
self.clip_peaklet_times(peaklets, start, end)
# Get hits outside peaklets, and store them separately.
# fully_contained is OK provided gap_threshold > extension,
# which is asserted inside strax.find_peaks.
lone_hits = hits[strax.fully_contained_in(hits, peaklets) == -1]
strax.integrate_lone_hits(
lone_hits,
records,
peaklets,
save_outside_hits=(self.config['peak_left_extension'],
self.config['peak_right_extension']),
n_channels=len(self.to_pe))
# Compute basic peak properties -- needed before natural breaks
strax.sum_waveform(peaklets, r, self.to_pe)
strax.compute_widths(peaklets)
# Split peaks using low-split natural breaks;
# see https://github.com/XENONnT/straxen/pull/45
# and https://github.com/AxFoundation/strax/pull/225
peaklets = strax.split_peaks(
peaklets,
r,
self.to_pe,
algorithm='natural_breaks',
threshold=self.natural_breaks_threshold,
split_low=True,
filter_wing_width=self.config['peak_split_filter_wing_width'],
min_area=self.config['peak_split_min_area'],
do_iterations=self.config['peak_split_iterations'])
# Saturation correction using non-saturated channels
# similar method used in pax
# see https://github.com/XENON1T/pax/pull/712
if self.config['saturation_correction_on']:
peak_saturation_correction(
r,
peaklets,
self.to_pe,
reference_length=self.config['saturation_reference_length'],
min_reference_length=self.
config['saturation_min_reference_length'])
# Compute tight coincidence level.
# Making this a separate plugin would
# (a) doing hitfinding yet again (or storing hits)
# (b) increase strax memory usage / max_messages,
# possibly due to its currently primitive scheduling.
hit_max_times = np.sort(hits['time'] +
hits['dt'] * hit_max_sample(records, hits))
peaklet_max_times = (
peaklets['time'] +
np.argmax(peaklets['data'], axis=1) * peaklets['dt'])
peaklets['tight_coincidence'] = get_tight_coin(
hit_max_times, peaklet_max_times,
self.config['tight_coincidence_window_left'],
self.config['tight_coincidence_window_right'])
if self.config['diagnose_sorting'] and len(r):
assert np.diff(r['time']).min(initial=1) >= 0, "Records not sorted"
assert np.diff(hits['time']).min(initial=1) >= 0, "Hits not sorted"
assert np.all(peaklets['time'][1:] >= strax.endtime(peaklets)[:-1]
), "Peaks not disjoint"
# Update nhits of peaklets:
counts = strax.touching_windows(hits, peaklets)
counts = np.diff(counts, axis=1).flatten()
counts += 1
peaklets['n_hits'] = counts
return dict(peaklets=peaklets, lone_hits=lone_hits)
