def compute(self, raw_records_coin_nv):
# Do not trust in DAQ + strax.baseline to leave the
# out-of-bounds samples to zero.
r = strax.raw_to_records(raw_records_coin_nv)
del raw_records_coin_nv
r = strax.sort_by_time(r)
strax.zero_out_of_bounds(r)
strax.baseline(r,
baseline_samples=self.config['baseline_samples_nv'],
flip=True)
strax.integrate(r)
strax.zero_out_of_bounds(r)
hits = strax.find_hits(
r, min_amplitude=self.config['hit_min_amplitude_nv'])
le, re = self.config['save_outside_hits_nv']
r = strax.cut_outside_hits(r,
hits,
left_extension=le,
right_extension=re)
strax.zero_out_of_bounds(r)
rlinks = strax.record_links(r)
r = clean_up_empty_records(r, rlinks, only_last=True)
return r
def compute(self, raw_records_coin_nv):
# Do not trust in DAQ + strax.baseline to leave the
# out-of-bounds samples to zero.
r = strax.raw_to_records(raw_records_coin_nv)
del raw_records_coin_nv
r = strax.sort_by_time(r)
strax.zero_out_of_bounds(r)
strax.baseline(r,
baseline_samples=self.baseline_samples,
flip=True)
if self.config['min_samples_alt_baseline_nv']:
m = r['pulse_length'] > self.config['min_samples_alt_baseline_nv']
if np.any(m):
# Correcting baseline after PMT saturated signals
r[m] = median_baseline(r[m])
strax.integrate(r)
strax.zero_out_of_bounds(r)
hits = strax.find_hits(r, min_amplitude=self.hit_thresholds)
le, re = self.config['save_outside_hits_nv']
r = strax.cut_outside_hits(r, hits, left_extension=le, right_extension=re)
strax.zero_out_of_bounds(r)
return r
def compute(self, chunk_i):
pre, current, post = self._chunk_paths(chunk_i)
if chunk_i == 0 and pre:
pre = False
print(f"There should be no {pre} dir for chunk 0: ignored")
records = np.concatenate(
([self._load_chunk(pre, kind='pre')] if pre else []) +
[self._load_chunk(current)] +
([self._load_chunk(post, kind='post')] if post else []))
strax.baseline(records)
strax.integrate(records)
if len(records):
# Convert time to time in ns since unix epoch.
# Ensure the offset is a whole digitizer sample
t0 = int(self.config["run_start_time"] * int(1e9))
dt = records[0]['dt']
t0 = dt * (t0 // dt)
records["time"] += t0
timespan_sec = (records[-1]['time'] - records[0]['time']) / 1e9
print(f'{chunk_i}: read {records.nbytes/1e6:.2f} MB '
f'({len(records)} records, '
f'{timespan_sec:.2f} sec) from readers')
else:
print(f'{chunk_i}: read an empty chunk!')
return records
def final_results(self):
records = self.record_buffer[:self.
blevel] # No copying the records from buffer
maska = records['time'] <= self.last_digitized_right * self.config[
'sample_duration']
records = records[maska]
records = strax.sort_by_time(records) # Do NOT remove this line
# strax.baseline(records) Will be done w/ pulse processing
strax.integrate(records)
# Yield an appropriate amount of stuff from the truth buffer
# and mark it as available for writing again
maskb = (
self.truth_buffer['fill'] &
# This condition will always be false if self.truth_buffer['t_first_photon'] == np.nan
((self.truth_buffer['t_first_photon'] <=
self.last_digitized_right * self.config['sample_duration']) |
# Hence, we need to use this trick to also save these cases (this
# is what we set the end time to for np.nans)
(np.isnan(self.truth_buffer['t_first_photon']) &
(self.truth_buffer['time'] <=
self.last_digitized_right * self.config['sample_duration']))))
truth = self.truth_buffer[maskb] # This is a copy, not a view!
# Careful here: [maskb]['fill'] = ... does not work
# numpy creates a copy of the array on the first index.
# The assignment then goes to the (unused) copy.
# ['fill'][maskb] leads to a view first, then the advanced
# assignment works into the original array as expected.
self.truth_buffer['fill'][maskb] = False
truth.sort(order='time')
# Return truth without 'fill' field
_truth = np.zeros(len(truth),
dtype=instruction_dtype + truth_extra_dtype)
for name in _truth.dtype.names:
_truth[name] = truth[name]
_truth['time'][~np.isnan(_truth['t_first_photon'])] = \
_truth['t_first_photon'][~np.isnan(_truth['t_first_photon'])].astype(int)
_truth.sort(order='time')
if self.config['detector'] == 'XENON1T':
yield dict(raw_records=records, truth=_truth)
else:
yield dict(
raw_records=records[records['channel'] < self.
config['channels_top_high_energy'][0]],
raw_records_he=records[
(records['channel'] >=
self.config['channels_top_high_energy'][0])
& (records['channel'] <=
self.config['channels_top_high_energy'][-1])],
raw_records_aqmon=records[records['channel'] == 800],
truth=_truth)
self.record_buffer[:np.sum(~maska)] = self.record_buffer[:self.blevel][
~maska]
self.blevel = np.sum(~maska)
def finish_results():
nonlocal results
records = np.concatenate(results)
# In strax data, records are always stored
# sorted, baselined and integrated
records = strax.sort_by_time(records)
strax.baseline(records)
strax.integrate(records)
results = []
return records
def compute(self, raw_records_nv, start, end):
strax.zero_out_of_bounds(raw_records_nv)
# First we have to split rr into records and lone records:
# Please note that we consider everything as a lone record which
# does not satisfy the coincidence requirement
intervals = coincidence(raw_records_nv,
self.config['coincidence_level_recorder_nv'],
self.config['resolving_time_recorder_nv'])
# Always save the first and last resolving_time nanoseconds (e.g. 600 ns) since we cannot guarantee the gap
# size to be larger. (We cannot use an OverlapingWindow plugin either since it requires disjoint objects.)
if len(intervals):
intervals_with_bounds = np.zeros((len(intervals) + 2, 2),
dtype=np.int64)
intervals_with_bounds[1:-1, :] = intervals
intervals_with_bounds[0, :] = start, min(
start + self.config['resolving_time_recorder_nv'],
intervals[0, 0])
intervals_with_bounds[-1, :] = max(
end - self.config['resolving_time_recorder_nv'],
intervals[-1, 1]), end
del intervals
else:
intervals_with_bounds = np.zeros((0, 2), dtype=np.int64)
neighbors = strax.record_links(raw_records_nv)
mask = pulse_in_interval(raw_records_nv, neighbors,
*np.transpose(intervals_with_bounds))
rr, lone_records = straxen.mask_and_not(raw_records_nv, mask)
# Compute some properties of the lone_records:
# We compute only for lone_records baseline etc. since
# raw_records_nv will be deleted, otherwise we could not change
# the settings and reprocess the data in case of raw_records_nv
lr = strax.raw_to_records(lone_records)
del lone_records
lr = strax.sort_by_time(lr)
strax.zero_out_of_bounds(lr)
strax.baseline(
lr,
baseline_samples=self.config['nbaseline_samples_lone_records_nv'],
flip=True)
strax.integrate(lr)
lrs, lr = compute_lone_records(lr, self.config['channel_map']['nveto'],
self.config['n_lone_records_nv'])
lrs['time'] = start
lrs['endtime'] = end
return {
'raw_records_coin_nv': rr,
'lone_raw_records_nv': lr,
'lone_raw_record_statistics_nv': lrs
}
def final_results(self, record_j):
records = self.record_buffer[:record_j] # Copy the records from buffer
records = strax.sort_by_time(
records) # Must keep this for sorted output
strax.baseline(records)
strax.integrate(records)
_truth = self.truth_buffer[self.truth_buffer['fill']]
# Return truth without 'fill' field
truth = np.zeros(len(_truth),
dtype=instruction_dtype + truth_extra_dtype)
for name in truth.dtype.names:
truth[name] = _truth[name]
return dict(raw_records=records, truth=truth)
def compute(self, chunk_i):
fp = self._chunk_path(chunk_i)
records = self._load_chunk(fp)
strax.baseline(records)
strax.integrate(records)
if len(records):
timespan_sec = (records[-1]['time'] - records[0]['time']) / 1e9
print(f'{chunk_i}: read {records.nbytes/1e6:.2f} MB '
f'({len(records)} records, '
f'{timespan_sec:.2f} live seconds)')
else:
print(f'{chunk_i}: read an empty chunk!')
return records
def compute(self, chunk_i):
pre, current, post = self._chunk_paths(chunk_i)
records = np.concatenate(
([self._load_chunk(pre, kind='pre')] if pre else []) +
[self._load_chunk(current)] +
([self._load_chunk(post, kind='post')] if post else []))
strax.baseline(records)
strax.integrate(records)
timespan_sec = (records[-1]['time'] - records[0]['time']) / 1e9
print(f'{chunk_i}: read {records.nbytes/1e6:.2f} MB '
f'({len(records)} records, '
f'{timespan_sec:.2f} sec) from readers')
return records
def compute(self, chunk_i):
pre, current, post = self._chunk_paths(chunk_i)
if chunk_i == 0 and pre:
pre = False
print(f"There should be no {pre} dir for chunk 0: ignored")
records = np.concatenate(
([self._load_chunk(pre, kind='pre')] if pre else []) +
[self._load_chunk(current)] +
([self._load_chunk(post, kind='post')] if post else []))
strax.baseline(records)
strax.integrate(records)
if len(records):
timespan_sec = (records[-1]['time'] - records[0]['time']) / 1e9
print(f'{chunk_i}: read {records.nbytes/1e6:.2f} MB '
f'({len(records)} records, '
f'{timespan_sec:.2f} sec) from readers')
else:
print(f'{chunk_i}: read an empty chunk!')
return records
def compute(self, raw_records_coin_nv):
# Do not trust in DAQ + strax.baseline to leave the
# out-of-bounds samples to zero.
r = strax.raw_to_records(raw_records_coin_nv)
del raw_records_coin_nv
r = strax.sort_by_time(r)
strax.zero_out_of_bounds(r)
strax.baseline(r, baseline_samples=self.baseline_samples, flip=True)
strax.integrate(r)
strax.zero_out_of_bounds(r)
hits = strax.find_hits(r, min_amplitude=self.hit_thresholds)
le, re = self.config['save_outside_hits_nv']
r = strax.cut_outside_hits(r,
hits,
left_extension=le,
right_extension=re)
strax.zero_out_of_bounds(r)
return r
def compute(self, raw_records, start, end):
if self.config['check_raw_record_overlaps']:
check_overlaps(raw_records, n_channels=3000)
# Throw away any non-TPC records; this should only happen for XENON1T
# converted data
raw_records = raw_records[
raw_records['channel'] < self.config['n_tpc_pmts']]
# Convert everything to the records data type -- adds extra fields.
r = strax.raw_to_records(raw_records)
del raw_records
# Do not trust in DAQ + strax.baseline to leave the
# out-of-bounds samples to zero.
# TODO: better to throw an error if something is nonzero
strax.zero_out_of_bounds(r)
strax.baseline(
r,
baseline_samples=self.config['baseline_samples'],
allow_sloppy_chunking=self.config['allow_sloppy_chunking'],
flip=True)
strax.integrate(r)
pulse_counts = count_pulses(r, self.config['n_tpc_pmts'])
pulse_counts['time'] = start
pulse_counts['endtime'] = end
if len(r) and self.hev_enabled:
r, r_vetoed, veto_regions = software_he_veto(
r,
self.to_pe,
end,
area_threshold=self.config['tail_veto_threshold'],
veto_length=self.config['tail_veto_duration'],
veto_res=self.config['tail_veto_resolution'],
pass_veto_extend=self.config['tail_veto_pass_extend'],
pass_veto_fraction=self.config['tail_veto_pass_fraction'],
max_veto_value=self.config['max_veto_value'])
# In the future, we'll probably want to sum the waveforms
# inside the vetoed regions, so we can still save the "peaks".
del r_vetoed
else:
veto_regions = np.zeros(0, dtype=strax.hit_dtype)
if len(r):
# Find hits
# -- before filtering,since this messes with the with the S/N
hits = strax.find_hits(r,
min_amplitude=straxen.hit_min_amplitude(
self.config['hit_min_amplitude']))
if self.config['pmt_pulse_filter']:
# Filter to concentrate the PMT pulses
strax.filter_records(r,
np.array(self.config['pmt_pulse_filter']))
le, re = self.config['save_outside_hits']
r = strax.cut_outside_hits(r,
hits,
left_extension=le,
right_extension=re)
# Probably overkill, but just to be sure...
strax.zero_out_of_bounds(r)
return dict(records=r,
pulse_counts=pulse_counts,
veto_regions=veto_regions)
def compute(self, raw_records_nv, start, end):
if self.config['check_raw_record_overlaps_nv']:
straxen.check_overlaps(raw_records_nv, n_channels=3000)
# Cover the case if we do not want to have any coincidence:
if self.config['coincidence_level_recorder_nv'] <= 1:
rr = raw_records_nv
lr = np.zeros(0, dtype=self.dtype['lone_raw_records_nv'])
lrs = np.zeros(0,
dtype=self.dtype['lone_raw_record_statistics_nv'])
return {
'raw_records_coin_nv': rr,
'lone_raw_records_nv': lr,
'lone_raw_record_statistics_nv': lrs
}
# Search for hits to define coincidence intervals:
temp_records = strax.raw_to_records(raw_records_nv)
temp_records = strax.sort_by_time(temp_records)
strax.zero_out_of_bounds(temp_records)
strax.baseline(temp_records,
baseline_samples=self.baseline_samples,
flip=True)
hits = strax.find_hits(temp_records, min_amplitude=self.hit_thresholds)
del temp_records
# First we have to split rr into records and lone records:
# Please note that we consider everything as a lone record which
# does not satisfy the coincidence requirement
intervals = find_coincidence(
hits, self.config['coincidence_level_recorder_nv'],
self.config['resolving_time_recorder_nv'],
self.config['pre_trigger_time_nv'])
del hits
# Always save the first and last resolving_time nanoseconds (e.g. 600 ns) since we cannot guarantee the gap
# size to be larger. (We cannot use an OverlapingWindow plugin either since it requires disjoint objects.)
if len(intervals):
intervals_with_bounds = np.zeros(len(intervals) + 2,
dtype=strax.time_fields)
intervals_with_bounds['time'][1:-1] = intervals['time']
intervals_with_bounds['endtime'][1:-1] = intervals['endtime']
intervals_with_bounds['time'][0] = start
intervals_with_bounds['endtime'][0] = min(
start + self.config['resolving_time_recorder_nv'],
intervals['time'][0])
intervals_with_bounds['time'][-1] = max(
end - self.config['resolving_time_recorder_nv'],
intervals['endtime'][-1])
intervals_with_bounds['endtime'][-1] = end
del intervals
else:
intervals_with_bounds = np.zeros((0, 2), dtype=strax.time_fields)
neighbors = strax.record_links(raw_records_nv)
mask = pulse_in_interval(
raw_records_nv,
neighbors,
intervals_with_bounds['time'],
intervals_with_bounds['endtime'],
)
rr, lone_records = straxen.mask_and_not(raw_records_nv, mask)
# Compute some properties of the lone_records:
# We compute only for lone_records baseline etc. since
# raw_records_nv will be deleted, otherwise we could not change
# the settings and reprocess the data in case of raw_records_nv
lr = strax.raw_to_records(lone_records)
del lone_records
lr = strax.sort_by_time(lr)
strax.zero_out_of_bounds(lr)
strax.baseline(lr, baseline_samples=self.baseline_samples, flip=True)
strax.integrate(lr)
lrs, lr = compute_lone_records(lr, self.config['channel_map']['nveto'],
self.config['n_lone_records_nv'])
lrs['time'] = start
lrs['endtime'] = end
return {
'raw_records_coin_nv': rr,
'lone_raw_records_nv': lr,
'lone_raw_record_statistics_nv': lrs
}
def pax_to_records(input_filename,
samples_per_record=strax.DEFAULT_RECORD_LENGTH):
"""Return pulse records array from pax zip input_filename"""
from pax import core # Pax is not a dependency
mypax = core.Processor(
'XENON1T',
config_dict=dict(
pax=dict(look_for_config_in_runs_db=False,
plugin_group_names=['input'],
encoder_plugin=None,
input_name=input_filename),
# Fast startup: skip loading big maps
WaveformSimulator=dict(s1_light_yield_map='placeholder_map.json',
s2_light_yield_map='placeholder_map.json',
s1_patterns_file=None,
s2_patterns_file=None)))
def get_events():
for e in mypax.get_events():
yield mypax.process_event(e)
# We loop over the events twice for convenience
# Yeah, this is probably not optimal
pulse_lengths = np.array(
[p.length for e in get_events() for p in e.pulses])
n_records = strax.records_needed(pulse_lengths, samples_per_record).sum()
records = np.zeros(n_records, dtype=strax.record_dtype(samples_per_record))
output_record_index = 0 # Record offset in data
for event in get_events():
for p in event.pulses:
n_records = strax.records_needed(p.length, samples_per_record)
for rec_i in range(n_records):
r = records[output_record_index]
r['time'] = (event.start_time + p.left * 10 +
rec_i * samples_per_record * 10)
r['channel'] = p.channel
r['pulse_length'] = p.length
r['record_i'] = rec_i
r['dt'] = 10
# How much are we storing in this record?
if rec_i != n_records - 1:
# There's more chunks coming, so we store a full chunk
n_store = samples_per_record
assert p.length > samples_per_record * (rec_i + 1)
else:
# Just enough to store the rest of the data
# Note it's not p.length % samples_per_record!!!
# (that would be zero if we have to store a full record)
n_store = p.length - samples_per_record * rec_i
assert 0 <= n_store <= samples_per_record
r['length'] = n_store
offset = rec_i * samples_per_record
r['data'][:n_store] = p.raw_data[offset:offset + n_store]
output_record_index += 1
mypax.shutdown()
# In strax data, records are always stored
# sorted, baselined and integrated
records = strax.sort_by_time(records)
strax.baseline(records)
strax.integrate(records)
return records
