def startup(self):
# Initialize buffer for numba signal finding routine.
# Although we're able to extend this buffer as needed, we must do so outside numba
# and it involves copyping data, so if you pick too small a buffer size you will hurt performance.
self.numba_signals_buffer = np.zeros(self.config['numba_signal_buffer_size'],
dtype=TriggerSignal.get_dtype())
# Initialize buffers for tallying pulses / coincidences
# Reason for +1 is again 'ghost' channels, see trigger.py
n_channels = self.trigger.pax_config['DEFAULT']['n_channels'] + 1
self.all_pulses_tally = np.zeros(n_channels, dtype=np.int)
self.lone_pulses_tally = np.zeros(n_channels, dtype=np.int)
self.coincidence_tally = np.zeros((n_channels, n_channels), dtype=np.int)
# Get conversion factor from ADC counts to pe for each pmt
# The 'ghost' PMT will have gain 1 always
self.gain_conversion_factors = np.array([adc_to_pe(self.trigger.pax_config['DEFAULT'], ch)
for ch in range(n_channels - 1)] +
[1])
# We must keep track of the next time to save the dark rate between batches, since a batch usually does not
# end exactly at a save time.
self.next_save_time = None
def WriteHits(self, peak, event, index):
''' Write out 1D histos for this peak '''
# hit_indices = {}
directory = self.outfile.mkdir(peak.type + "_" + str(index))
hists = []
pulses_to_write = {}
hitlist = {}
for hit in peak.hits:
hitdict = {
"found_in_pulse":
hit[3],
"area":
hit[0],
"channel":
hit[2],
"center":
hit[1] / 10,
"left":
hit[7] - 1,
"right":
hit[10],
"max_index":
hit[5],
"height":
hit[4] / dsputils.adc_to_pe(
self.config, hit[2], use_reference_gain=True)
}
if hitdict['channel'] not in hitlist:
hitlist[hitdict['channel']] = []
pulses_to_write[hitdict['channel']] = []
hitlist[hitdict['channel']].append(hitdict)
if hitdict['found_in_pulse'] not in pulses_to_write[
hitdict['channel']]:
pulses_to_write[hitdict['channel']].append(
hitdict['found_in_pulse'])
# Now we should have pulses_to_write with which
# pulses to plot per channel and
# hitlist with a list of all hit properties to add
for channel, pulselist in pulses_to_write.items():
leftbound = -1
rightbound = -1
# Needs an initial scan to find histogram range
for pulseid in pulselist:
if leftbound == -1 or event.pulses[pulseid].left < leftbound:
leftbound = event.pulses[pulseid].left
if rightbound == -1 or event.pulses[pulseid].right > rightbound:
rightbound = event.pulses[pulseid].right
# Make and book the histo. Put into hists so doesn't get overwritten
histname = "%s_%i_channel_%i" % (peak.type, index, channel)
histtitle = "Channel %i in %s[%i]" % (channel, peak.type, index)
c = ROOT.TCanvas(histname, "")
h = ROOT.TH1F(histname, histtitle, int(rightbound - leftbound),
float(leftbound), float(rightbound))
# Now put the bin values in the histogram
for pulseid in pulselist:
pulse = event.pulses[pulseid]
w = (self.config['digitizer_reference_baseline'] +
pulse.baseline - pulse.raw_data.astype(np.float64))
for i, sample in enumerate(w):
h.SetBinContent(int(i + pulse.left - leftbound), sample)
h.SetStats(0)
h.GetXaxis().SetTitle("Time [samples]")
h.GetYaxis().SetTitleOffset(0.8)
h.GetYaxis().SetTitleSize(0.05)
h.GetXaxis().SetTitleOffset(0.8)
h.GetXaxis().SetTitleSize(0.05)
h.GetYaxis().SetTitle("ADC Reading (baseline corrected)")
c.cd()
h.Draw()
plist = {"x": [], "y": []}
for i, hitdict in enumerate(hitlist[channel]):
baseline = ROOT.TLine(hitdict['left'], pulse.baseline,
hitdict['right'], pulse.baseline)
leftline = ROOT.TLine(hitdict['left'], 0, hitdict['left'],
hitdict['height'])
rightline = ROOT.TLine(hitdict['right'], 0, hitdict['right'],
hitdict['height'])
plist['x'].append(hitdict['center'])
plist['y'].append(hitdict['height'])
leftline.SetLineStyle(2)
rightline.SetLineStyle(2)
leftline.SetLineColor(2)
rightline.SetLineColor(2)
baseline.SetLineStyle(2)
baseline.SetLineColor(4)
baseline.Draw("same")
leftline.Draw("same")
rightline.Draw("same")
label = "hit " + str(i) + "({:.2f} p.e.)".format(
hitdict['area'])
text = ROOT.TText(hitdict['center'], hitdict['height'], label)
text.SetTextColor(2)
text.SetTextSize(0.03)
text.Draw("same")
c.Update()
hists.append({
"lline": leftline,
"text": text,
"rline": rightline,
"bline": baseline
})
c.cd()
polymarker = ROOT.TPolyMarker()
polymarker.SetMarkerStyle(23)
polymarker.SetMarkerColor(2)
polymarker.SetMarkerSize(1.1)
polymarker.SetPolyMarker(len(plist['x']), np.array(plist['x']),
np.array(plist['y']))
polymarker.Draw("same")
c.Update()
hists.append({"poly": polymarker, "hist": h, "c": c})
directory.cd()
c.Write()
return hists
def transform_event(self, event):
tpc_channels = np.array(self.config['channels_in_detector']['tpc'])
# Boolean array, tells us which pulses are saturated
is_saturated = np.array([
p.maximum >= self.reference_baseline - p.baseline - 0.5
for p in event.pulses
])
for pulse_i, pulse in enumerate(event.pulses):
# Consider only saturated pulses in the TPC
if not is_saturated[pulse_i] or pulse.channel not in tpc_channels:
continue
# Where is the current pulse saturated?
saturated = pulse.raw_data <= 0 # Boolean array, True if sample is saturated
_where_saturated = np.where(saturated)[0]
try:
first_saturated = _where_saturated.min()
last_saturated = _where_saturated.max()
except (ValueError, RuntimeError, TypeError, NameError):
continue
# Select a reference region just before the start of the saturated region
reference_slice = slice(
max(0,
first_saturated - self.config['reference_region_samples']),
first_saturated)
# Find all pulses in TPC channels that overlap with the saturated & reference region
other_pulses = [
p for i, p in enumerate(event.pulses)
if p.left < last_saturated +
pulse.left and p.right > pulse.left and not is_saturated[i]
and p.channel in tpc_channels
]
if not len(other_pulses):
# Rare case where no other pulses available, one channel going crazy?
continue
# Compute the (gain-weighted) sum waveform of the non-saturated pulses
min_left = min([p.left for p in other_pulses + [pulse]])
max_right = max([p.right for p in other_pulses + [pulse]])
sumw = np.zeros(max_right - min_left + 1)
for p in other_pulses:
offset = p.left - min_left
sumw[offset:offset + len(p.raw_data)] += self.waveform_in_pe(p)
# Crop it to include just the part that overlaps with this pulse
offset = pulse.left - min_left
sumw = sumw[offset:offset + len(pulse.raw_data)]
# Compute the ratio of this channel's waveform / the nonsaturated waveform in the reference region
w = self.waveform_in_pe(pulse)
ratio = w[reference_slice].sum() / sumw[reference_slice].sum()
# not < is preferred over >, since it will catch nan
if not ratio < self.config.get('min_reference_area_ratio', 1):
# The pulse is saturated, but insufficient information is available in the other channels
# to reliably reconstruct it
continue
if len(w[reference_slice][w[reference_slice] > 1]
) < self.config['reference_region_samples_treshold']:
# the pulse is saturated, but there are not enough reference samples to get a good ratio
# This actually distinguished between S1 and S2 and will only correct S2 signals
continue
# Reconstruct the waveform in the saturated region according to this ratio.
# The waveform should never be reduced due to this (then the correction is making things worse)
w[saturated] = np.clip(sumw[saturated] * ratio, w[saturated],
float('inf'))
# Convert back to raw ADC counts and store the corrected waveform
# Note this changes the type of pulse.w from int16 to float64: we don't have a choice,
# int16 probably can't contain the large amplitudes we may be putting in.
# As long as the raw data isn't saved again after applying this correction, this should be no problem
# (as in later code converting to floats is anyway the first step).
w /= adc_to_pe(self.config, pulse.channel)
w = self.reference_baseline - w - pulse.baseline
pulse.raw_data = w
return event
def waveform_in_pe(self, p):
"""Return waveform in pe/bin above baseline of a pulse"""
w = self.reference_baseline - p.raw_data.astype(np.float) - p.baseline
w *= adc_to_pe(self.config, p.channel)
return w
def transform_event(self, event):
dt = self.config['sample_duration']
hits_per_pulse = []
left_extension = self.config['left_extension'] // dt
right_extension = self.config['right_extension'] // dt
# Allocate numpy arrays to hold numba hitfinder results
# -1 is a placeholder for values that should never appear (0 would be bad as it often IS a possible value)
hit_bounds_buffer = -1 * np.ones((self.max_hits_per_pulse, 2), dtype=np.int64)
hits_buffer = np.zeros(self.max_hits_per_pulse, dtype=datastructure.Hit.get_dtype())
for pulse_i, pulse in enumerate(event.pulses):
start = pulse.left
stop = pulse.right
channel = pulse.channel
pmt_gain = self.config['gains'][channel]
# Check the pulse properties have been computed
if np.isnan(pulse.minimum):
raise RuntimeError("Attempt to perform hitfinding on pulses whose properties have not been computed!")
# Retrieve waveform as floats: needed to subtract baseline (which can be in-between ADC counts)
w = pulse.raw_data.astype(np.float64)
# Subtract the baseline and invert(so hits point up from baseline)
w = self.reference_baseline - w
w -= pulse.baseline
# Don't do hitfinding in dead channels, pulse property computation was enough
# Could refactor pulse property computation to separate plugin,
# but that would mean waveform has to be converted to floats twice
if pmt_gain == 0:
continue
# Compute hitfinder threshold to use
# Rounding down is ok, since hitfinder uses >, not >= for threshold crossing testing.
pulse.hitfinder_threshold = int(max(self.config['height_over_noise_threshold'] * pulse.noise_sigma,
self.config['absolute_adc_counts_threshold'],
- self.config['height_over_min_threshold'] * pulse.minimum))
if self.always_find_single_hit:
# The config specifies a single range to integrate. Useful for gain calibration
hit_bounds_buffer[0] = self.always_find_single_hit
n_hits_found = 1
else:
# Call the numba hit finder -- see its docstring for description
n_hits_found = dsputils.find_intervals_above_threshold(w,
threshold=float(pulse.hitfinder_threshold),
result_buffer=hit_bounds_buffer)
# Only view the part of hit_bounds_buffer that contains hits found in this event
# The rest of hit_bounds_buffer contains -1's or stuff from previous pulses
pulse.n_hits_found = n_hits_found
hit_bounds_found = hit_bounds_buffer[:n_hits_found]
if self.always_find_single_hit:
central_bounds = hit_bounds_found
else:
# Extend the boundaries of each hit, to be sure we integrate everything.
# The original bounds are preserved: they are used in clustering
central_bounds = hit_bounds_found.copy()
dsputils.extend_intervals(w, hit_bounds_found, left_extension, right_extension)
# If no hits were found, this is a noise pulse: update the noise pulse count
if n_hits_found == 0:
event.noise_pulses_in[channel] += 1
# Don't 'continue' to the next pulse! There's stuff left to do!
elif n_hits_found >= self.max_hits_per_pulse:
self.log.debug("Pulse %s-%s in channel %s has more than %s hits. "
"This usually indicates a zero-length encoding breakdown after a very large S2. "
"Further hits in this pulse have been ignored." % (start, stop, channel,
self.max_hits_per_pulse))
# Store the found hits in the datastructure
# Convert area, noise_sigma and height from adc counts -> pe
adc_to_pe = dsputils.adc_to_pe(self.config, channel)
noise_sigma_pe = pulse.noise_sigma * adc_to_pe
# If the DAQ pulse was ADC-saturated (clipped), the raw waveform dropped to 0,
# i.e. we went digitizer_reference_baseline above the reference baseline
# i.e. we went digitizer_reference_baseline - pulse.baseline above baseline
# 0.5 is needed to avoid floating-point rounding errors to cause saturation not to be reported
# Somehow happens only when you use simulated data -- apparently np.clip rounds slightly different
saturation_threshold = self.reference_baseline - pulse.baseline - 0.5
build_hits(w, hit_bounds_found, hits_buffer,
adc_to_pe, channel, noise_sigma_pe, dt, start, pulse_i, saturation_threshold, central_bounds)
hits = hits_buffer[:n_hits_found].copy()
hits_per_pulse.append(hits)
if len(hits_per_pulse):
event.all_hits = np.concatenate(hits_per_pulse)
if not self.always_find_single_hit:
# Remove hits with 0 or negative area (very rare, but possible due to rigid integration bound)
# In always-find-single-hit mode (for PMT calibrations) this is undesirable
event.all_hits = event.all_hits[event.all_hits['area'] > 0]
self.log.debug("Found %d hits in %d pulses" % (len(event.all_hits), len(event.pulses)))
else:
self.log.warning("Event has no pulses??!")
return event
def transform_event(self, event):
tpc_channels = np.array(self.config['channels_in_detector']['tpc'])
# Boolean array, tells us which pulses are saturated
is_saturated = np.array([
p.maximum >= self.reference_baseline - p.baseline - 0.5
for p in event.pulses
])
for pulse_i, pulse in enumerate(event.pulses):
# Consider only saturated pulses in the TPC
if not is_saturated[pulse_i] or pulse.channel not in tpc_channels:
continue
# Where is the current pulse saturated?
saturated = pulse.raw_data <= 0 # Boolean array, True if sample is saturated
_where_saturated_all = np.where(saturated)[0]
# Split saturation if there is long enough non-saturated samples in between
_where_saturated_diff = np.diff(_where_saturated_all, n=1)
_where_saturated_diff = np.where(
_where_saturated_diff > self.config['reference_region_samples']
)[0]
_where_saturated_list = np.split(_where_saturated_all,
_where_saturated_diff + 1)
# Find all pulses in TPC channels that overlap with the saturated & reference region
other_pulses = [
p for i, p in enumerate(event.pulses)
if p.left < pulse.right and p.right > pulse.left
and not is_saturated[i] and p.channel in tpc_channels and
p.channel not in self.config['large_after_pulsing_channels']
]
if not len(other_pulses):
# Rare case where no other pulses available, one channel going crazy?
continue
for peak_i, _where_saturated in enumerate(_where_saturated_list):
try:
first_saturated = _where_saturated.min()
last_saturated = _where_saturated.max()
except (ValueError, RuntimeError, TypeError, NameError):
continue
# Select a reference region just before the start of the saturated region
reference_slice = slice(
max(
0, first_saturated -
self.config['reference_region_samples']),
first_saturated)
# Compute the (gain-weighted) sum waveform of the non-saturated pulses
min_left = min([p.left for p in other_pulses + [pulse]])
max_right = max([p.right for p in other_pulses + [pulse]])
sumw = np.zeros(max_right - min_left + 1)
for p in other_pulses:
offset = p.left - min_left
sumw[offset:offset +
len(p.raw_data)] += self.waveform_in_pe(p)
# Crop it to include just the part that overlaps with this pulse
offset = pulse.left - min_left
sumw = sumw[offset:offset + len(pulse.raw_data)]
# Compute the ratio of this channel's waveform / the nonsaturated waveform in the reference region
w = self.waveform_in_pe(pulse)
if len(sumw[reference_slice][sumw[reference_slice] > 1]) \
< self.config['reference_region_samples_treshold']:
# the pulse is saturated, but there are not enough reference samples to get a good ratio
# This actually distinguished between S1 and S2 and will only correct S2 signals
continue
ratio = w[reference_slice].sum() / sumw[reference_slice].sum()
# not < is preferred over >, since it will catch nan
if not ratio < self.config.get('min_reference_area_ratio', 1):
# The pulse is saturated, but insufficient information is available in the other channels
# to reliably reconstruct it
continue
if len(w[reference_slice][w[reference_slice] > 1]
) < self.config['reference_region_samples_treshold']:
# the pulse is saturated, but there are not enough reference samples to get a good ratio
# This actually distinguished between S1 and S2 and will only correct S2 signals
continue
# Finding individual section of wf for each peak
# First end before the reference region of next peak
if peak_i + 1 == len(_where_saturated_list):
end = len(w)
else:
end = _where_saturated_list[
peak_i +
1][0] - self.config['reference_region_samples']
# Second end before the first upwards turning point
v = sumw[last_saturated:end]
conv = np.ones(self.config['convolution_length']
) / self.config['convolution_length']
v = np.convolve(conv, v, mode='same')
dv = np.diff(v, n=1)
# Choose +2 pe/ns instead 0 to avoid ending on the flat waveform
turning_point = np.where((np.hstack((dv, -10)) > 2)
& (np.hstack((10, dv)) <= 2))[0]
if len(turning_point) > 0:
end = last_saturated + turning_point[0]
# Reconstruct the waveform in the saturated region according to this ratio.
# The waveform should never be reduced due to this (then the correction is making things worse)
saturated_to_correct = np.arange(int(first_saturated),
int(end))
w[saturated_to_correct] = np.clip(
sumw[saturated_to_correct] * ratio, 0, float('inf'))
# Convert back to raw ADC counts and store the corrected waveform
# Note this changes the type of pulse.w from int16 to float64: we don't have a choice,
# int16 probably can't contain the large amplitudes we may be putting in.
# As long as the raw data isn't saved again after applying this correction, this should be no problem
# (as in later code converting to floats is anyway the first step).
w /= adc_to_pe(self.config, pulse.channel)
w = self.reference_baseline - w - pulse.baseline
pulse.raw_data = w
return event
def transform_event(self, event):
tpc_channels = np.array(self.config['channels_in_detector']['tpc'])
# Boolean array, tells us which pulses are saturated
is_saturated = np.array([p.maximum >= self.reference_baseline - p.baseline - 0.5
for p in event.pulses])
for pulse_i, pulse in enumerate(event.pulses):
# Consider only saturated pulses in the TPC
if not is_saturated[pulse_i] or pulse.channel not in tpc_channels:
continue
# Where is the current pulse saturated?
saturated = pulse.raw_data <= 0 # Boolean array, True if sample is saturated
_where_saturated = np.where(saturated)[0]
try:
first_saturated = _where_saturated.min()
last_saturated = _where_saturated.max()
except (ValueError, RuntimeError, TypeError, NameError):
continue
# Select a reference region just before the start of the saturated region
reference_slice = slice(max(0, first_saturated - self.config['reference_region_samples']),
first_saturated)
# Find all pulses in TPC channels that overlap with the saturated & reference region
other_pulses = [p for i, p in enumerate(event.pulses)
if p.left < last_saturated + pulse.left and p.right > pulse.left and
not is_saturated[i] and
p.channel in tpc_channels]
if not len(other_pulses):
# Rare case where no other pulses available, one channel going crazy?
continue
# Compute the (gain-weighted) sum waveform of the non-saturated pulses
min_left = min([p.left for p in other_pulses + [pulse]])
max_right = max([p.right for p in other_pulses + [pulse]])
sumw = np.zeros(max_right - min_left + 1)
for p in other_pulses:
offset = p.left - min_left
sumw[offset:offset + len(p.raw_data)] += self.waveform_in_pe(p)
# Crop it to include just the part that overlaps with this pulse
offset = pulse.left - min_left
sumw = sumw[offset:offset + len(pulse.raw_data)]
# Compute the ratio of this channel's waveform / the nonsaturated waveform in the reference region
w = self.waveform_in_pe(pulse)
ratio = w[reference_slice].sum()/sumw[reference_slice].sum()
# not < is preferred over >, since it will catch nan
if not ratio < self.config.get('min_reference_area_ratio', 1):
# The pulse is saturated, but insufficient information is available in the other channels
# to reliably reconstruct it
continue
if len(w[reference_slice][w[reference_slice] > 1]) < self.config['reference_region_samples_treshold']:
# the pulse is saturated, but there are not enough reference samples to get a good ratio
# This actually distinguished between S1 and S2 and will only correct S2 signals
continue
# Reconstruct the waveform in the saturated region according to this ratio.
# The waveform should never be reduced due to this (then the correction is making things worse)
w[saturated] = np.clip(sumw[saturated] * ratio, w[saturated], float('inf'))
# Convert back to raw ADC counts and store the corrected waveform
# Note this changes the type of pulse.w from int16 to float64: we don't have a choice,
# int16 probably can't contain the large amplitudes we may be putting in.
# As long as the raw data isn't saved again after applying this correction, this should be no problem
# (as in later code converting to floats is anyway the first step).
w /= adc_to_pe(self.config, pulse.channel)
w = self.reference_baseline - w - pulse.baseline
pulse.raw_data = w
return event
def plot_event(self, event):
dt = self.config['sample_duration']
# Configurable limits
channels_start = self.config.get(
'channel_start', min(self.config['channels_in_detector']['tpc']))
channels_end = self.config.get(
'channel_start', max(self.config['channels_in_detector']['tpc']))
t_start = self.config.get('t_start', 0 * units.us)
t_end = self.config.get('t_end', event.duration())
fig = plt.figure(figsize=(self.size_multiplier * 4,
self.size_multiplier * 2))
ax = fig.gca(projection='3d')
global_max_amplitude = 0
for pulse in event.pulses:
# Take only channels in the range we want to plot
if not channels_start <= pulse.channel <= channels_end:
continue
# Take only pulses that fall (at least partially) in the time window we want to plot
if pulse.right * dt < t_start or pulse.left * dt > t_end:
continue
w = self.config[
'digitizer_reference_baseline'] + pulse.baseline - pulse.raw_data.astype(
np.float64)
w *= dsputils.adc_to_pe(self.config,
pulse.channel,
use_reference_gain=True)
if self.config['log_scale']:
# This will still give nan's if waveform drops below 1 pe_nominal / bin...
# TODO: So... will it crash? or just fall outside range?
w = np.log10(1 + w)
# We need to keep track of this, apparently gca can't scale itself?
global_max_amplitude = max(np.max(w), global_max_amplitude)
ax.plot(np.linspace(pulse.left, pulse.right, pulse.length) * dt /
units.us,
pulse.channel * np.ones(pulse.length),
zs=w,
zdir='z',
label=str(pulse.channel))
# Plot the sum waveform
w = event.get_sum_waveform('tpc').samples[int(t_start /
dt):int(t_end / dt) + 1]
ax.plot(np.linspace(t_start, t_end, len(w)) / units.us,
(channels_end + 1) * np.ones(len(w)),
zs=w * global_max_amplitude / np.max(w),
zdir='z',
label='Tpc')
ax.set_xlabel('Time [$\mu$s]')
ax.set_xlim3d(t_start / units.us, t_end / units.us)
ax.set_ylabel('Channel number')
ax.set_ylim3d(channels_start, channels_end)
zlabel = 'Pulse height [pe_nominal / %d ns]' % (
self.config['sample_duration'] / units.ns)
if self.config['log_scale']:
zlabel = 'Log10 1 + ' + zlabel
ax.set_zlabel(zlabel)
ax.set_zlim3d(0, global_max_amplitude)
def transform_event(self, event):
tpc_channels = np.array(self.config['channels_in_detector']['tpc'])
# Boolean array, tells us which pulses are saturated
is_saturated = np.array([p.maximum >= self.reference_baseline - p.baseline - 0.5
for p in event.pulses])
for pulse_i, pulse in enumerate(event.pulses):
# Consider only saturated pulses in the TPC
if not is_saturated[pulse_i] or pulse.channel not in tpc_channels:
continue
# Where is the current pulse saturated?
saturated = pulse.raw_data <= 0 # Boolean array, True if sample is saturated
_where_saturated_all = np.where(saturated)[0]
# Split saturation if there is long enough non-saturated samples in between
_where_saturated_diff = np.diff(_where_saturated_all, n=1)
_where_saturated_diff = np.where(_where_saturated_diff > self.config['reference_region_samples'])[0]
_where_saturated_list = np.split(_where_saturated_all, _where_saturated_diff+1)
# Find all pulses in TPC channels that overlap with the saturated & reference region
other_pulses = [p for i, p in enumerate(event.pulses)
if p.left < pulse.right and p.right > pulse.left and
not is_saturated[i] and
p.channel in tpc_channels and
p.channel not in self.config['large_after_pulsing_channels']]
if not len(other_pulses):
# Rare case where no other pulses available, one channel going crazy?
continue
for peak_i, _where_saturated in enumerate(_where_saturated_list):
try:
first_saturated = _where_saturated.min()
last_saturated = _where_saturated.max()
except (ValueError, RuntimeError, TypeError, NameError):
continue
# Select a reference region just before the start of the saturated region
reference_slice = slice(max(0, first_saturated - self.config['reference_region_samples']),
first_saturated)
# Compute the (gain-weighted) sum waveform of the non-saturated pulses
min_left = min([p.left for p in other_pulses + [pulse]])
max_right = max([p.right for p in other_pulses + [pulse]])
sumw = np.zeros(max_right - min_left + 1)
for p in other_pulses:
offset = p.left - min_left
sumw[offset:offset + len(p.raw_data)] += self.waveform_in_pe(p)
# Crop it to include just the part that overlaps with this pulse
offset = pulse.left - min_left
sumw = sumw[offset:offset + len(pulse.raw_data)]
# Compute the ratio of this channel's waveform / the nonsaturated waveform in the reference region
w = self.waveform_in_pe(pulse)
if len(sumw[reference_slice][sumw[reference_slice] > 1]) \
< self.config['reference_region_samples_treshold']:
# the pulse is saturated, but there are not enough reference samples to get a good ratio
# This actually distinguished between S1 and S2 and will only correct S2 signals
continue
ratio = w[reference_slice].sum()/sumw[reference_slice].sum()
# not < is preferred over >, since it will catch nan
if not ratio < self.config.get('min_reference_area_ratio', 1):
# The pulse is saturated, but insufficient information is available in the other channels
# to reliably reconstruct it
continue
if len(w[reference_slice][w[reference_slice] > 1]) < self.config['reference_region_samples_treshold']:
# the pulse is saturated, but there are not enough reference samples to get a good ratio
# This actually distinguished between S1 and S2 and will only correct S2 signals
continue
# Finding individual section of wf for each peak
# First end before the reference region of next peak
if peak_i+1 == len(_where_saturated_list):
end = len(w)
else:
end = _where_saturated_list[peak_i+1][0]-self.config['reference_region_samples']
# Second end before the first upwards turning point
v = sumw[last_saturated: end]
conv = np.ones(self.config['convolution_length'])/self.config['convolution_length']
v = np.convolve(conv, v, mode='same')
dv = np.diff(v, n=1)
# Choose +2 pe/ns instead 0 to avoid ending on the flat waveform
turning_point = np.where((np.hstack((dv, -10)) > 2) & (np.hstack((10, dv)) <= 2))[0]
if len(turning_point) > 0:
end = last_saturated + turning_point[0]
# Reconstruct the waveform in the saturated region according to this ratio.
# The waveform should never be reduced due to this (then the correction is making things worse)
saturated_to_correct = np.arange(int(first_saturated), int(end))
w[saturated_to_correct] = np.clip(sumw[saturated_to_correct] * ratio, 0, float('inf'))
# Convert back to raw ADC counts and store the corrected waveform
# Note this changes the type of pulse.w from int16 to float64: we don't have a choice,
# int16 probably can't contain the large amplitudes we may be putting in.
# As long as the raw data isn't saved again after applying this correction, this should be no problem
# (as in later code converting to floats is anyway the first step).
w /= adc_to_pe(self.config, pulse.channel)
w = self.reference_baseline - w - pulse.baseline
pulse.raw_data = w
return event
def waveform_in_pe(self, p):
"""Return waveform in pe/bin above baseline of a pulse"""
w = self.reference_baseline - p.raw_data.astype(np.float) - p.baseline
w *= adc_to_pe(self.config, p.channel)
return w
def transform_event(self, event):
if self.make_diagnostic_plots == 'never':
return event
# Get the pulse-to-hit mapping
# Note this relies on the hits being sorted by found_in_pulse
# (probably shouldn't have rolled our own fake pandas for recarrays...)
self.log.debug("Reconstructing hit/pulse mapping")
hits_in_pulse = dict_group_by(event.all_hits, 'found_in_pulse')
for pulse_i, pulse in enumerate(
tqdm(event.pulses, desc='Making hitfinder diagnostic plots')):
# Get the hits that were found in this pulse
hits = hits_in_pulse.get(
pulse_i, np.array([], datastructure.Hit.get_dtype()))
# Reconstruct some variables we had in the hitfinder. Some code duplication unfortunately...
# that's the price we pay for having diagnostic plotting cleanly separated from the hitfinder.
start = pulse.left
stop = pulse.right
hit_bounds_found = np.vstack((hits['left'], hits['right'])).T
hit_bounds_found -= start
w = self.reference_baseline - pulse.raw_data.astype(
np.float64) - pulse.baseline
channel = pulse.channel
adc_to_pe = dsputils.adc_to_pe(self.config, channel)
noise_sigma_pe = pulse.noise_sigma * adc_to_pe
threshold = pulse.hitfinder_threshold
saturation_threshold = self.reference_baseline - pulse.baseline - 0.5
is_saturated = pulse.maximum >= saturation_threshold
# Do we need to show this pulse? If not: continue
if self.make_diagnostic_plots == 'tricky cases':
# Always show pulse if noise level is very high
if noise_sigma_pe < 0.5:
if len(hit_bounds_found) == 0:
# Show pulse if it nearly went over threshold
if not pulse.maximum > 0.8 * threshold:
continue
else:
# Show pulse if any of its hit nearly didn't go over threshold
if not any([
event.all_hits[-(i + 1)].height <
1.2 * threshold * adc_to_pe
for i in range(len(hit_bounds_found))
]):
continue
elif self.make_diagnostic_plots == 'no hits':
if len(hit_bounds_found) != 0:
continue
elif self.make_diagnostic_plots == 'baseline shifts':
if abs(pulse.baseline_increase) < 10:
continue
elif self.make_diagnostic_plots == 'hits only':
if len(hit_bounds_found) == 0:
continue
elif self.make_diagnostic_plots == 'saturated':
if not is_saturated:
continue
elif self.make_diagnostic_plots == 'negative':
# Select only pulses which had hits whose area and or height were originally negative
# (the hitfinder helpfully capped them at 1e-9, so they're not actually negative...)
if not (len(hits) and (np.any(hits['area'] < 1e-6)
or np.any(hits['height'] < 1e-6))):
continue
elif self.make_diagnostic_plots != 'always':
raise ValueError("Invalid make_diagnostic_plots option: %s!" %
self.make_diagnostic_plots)
plt.figure(figsize=(14, 10))
data_for_title = (event.event_number, start, stop, channel)
plt.title('Event %s, pulse %d-%d, Channel %d' % data_for_title)
ax1 = plt.gca()
ax2 = ax1.twinx()
ax1.set_position((.1, .1, .6, .85))
ax2.set_position((.1, .1, .6, .85))
ax1.set_xlabel("Sample number (%s ns)" % event.sample_duration)
ax1.set_ylabel("ADC counts above baseline")
ax2.set_ylabel("pe / sample")
# Plot the signal and noise levels
ax1.plot(w, drawstyle='steps-mid', label='Data')
ax1.plot(np.ones_like(w) * threshold,
'--',
label='Threshold',
color='red')
ax1.plot(np.ones_like(w) * pulse.noise_sigma,
':',
label='Noise level',
color='gray')
ax1.plot(np.ones_like(w) * pulse.minimum,
'--',
label='Minimum',
color='orange')
# Mark the hit ranges & center of gravity point
for hit_i, hit in enumerate(hit_bounds_found):
ax1.axvspan(hit[0] - 0.5, hit[1] + 0.5, color='red', alpha=0.2)
# Make sure the y-scales match
ax2.set_ylim(ax1.get_ylim()[0] * adc_to_pe,
ax1.get_ylim()[1] * adc_to_pe)
# Add pulse / hit information
if len(hits) != 0:
largest_hit = hits[np.argmax(hits['area'])]
plt.figtext(0.75,
0.98,
dedent("""
Pulse maximum: {pulse.maximum:.5g}
Pulse minimum: {pulse.minimum:.5g}
(both in ADCc above baseline)
Pulse baseline: {pulse.baseline}
(in ADCc above reference baseline)
Baseline increase: {pulse.baseline_increase:.2f}
Gain in this PMT: {gain:.3g}
Noise level: {pulse.noise_sigma:.2f} ADCc
Hitfinder threshold: {pulse.hitfinder_threshold} ADCc
Largest hit info ({left}-{right}):
Area: {hit_area:.5g} pe
Height: {hit_height:.4g} pe
Saturated samples: {hit_n_saturated}
""".format(
pulse=pulse,
gain=self.config['gains'][pulse.channel],
left=largest_hit['left'] - pulse.left,
right=largest_hit['right'] - pulse.left,
hit_area=largest_hit['area'],
hit_height=largest_hit['height'],
hit_n_saturated=largest_hit['n_saturated'])),
fontsize=14,
verticalalignment='top')
# Finish the plot, save, close
leg = ax1.legend()
leg.get_frame().set_alpha(0.5)
plt.savefig(
os.path.join(
self.make_diagnostic_plots_in,
'event%04d_pulse%05d-%05d_ch%03d.png' % data_for_title))
plt.xlim(0, len(pulse.raw_data))
plt.close()
return event
def transform_event(self, event):
dt = self.config['sample_duration']
hits_per_pulse = []
left_extension = self.config['left_extension'] // dt
right_extension = self.config['right_extension'] // dt
# Allocate numpy arrays to hold numba hitfinder results
# -1 is a placeholder for values that should never appear (0 would be bad as it often IS a possible value)
hit_bounds_buffer = -1 * np.ones(
(self.max_hits_per_pulse, 2), dtype=np.int64)
hits_buffer = np.zeros(self.max_hits_per_pulse,
dtype=datastructure.Hit.get_dtype())
for pulse_i, pulse in enumerate(event.pulses):
start = pulse.left
stop = pulse.right
channel = pulse.channel
pmt_gain = self.config['gains'][channel]
# Check the pulse properties have been computed
if np.isnan(pulse.minimum):
raise RuntimeError(
"Attempt to perform hitfinding on pulses whose properties have not been computed!"
)
# Retrieve waveform as floats: needed to subtract baseline (which can be in-between ADC counts)
w = pulse.raw_data.astype(np.float64)
# Subtract the baseline and invert(so hits point up from baseline)
w = self.reference_baseline - w
w -= pulse.baseline
# Don't do hitfinding in dead channels, pulse property computation was enough
# Could refactor pulse property computation to separate plugin,
# but that would mean waveform has to be converted to floats twice
if pmt_gain == 0:
continue
# Compute hitfinder threshold to use
# Rounding down is ok, since hitfinder uses >, not >= for threshold crossing testing.
pulse.hitfinder_threshold = int(
max(
self.config['height_over_noise_threshold'] *
pulse.noise_sigma,
self.config['absolute_adc_counts_threshold'],
-self.config['height_over_min_threshold'] * pulse.minimum))
if self.always_find_single_hit:
# The config specifies a single range to integrate. Useful for gain calibration
hit_bounds_buffer[0] = self.always_find_single_hit
n_hits_found = 1
else:
# Call the numba hit finder -- see its docstring for description
n_hits_found = dsputils.find_intervals_above_threshold(
w,
threshold=float(pulse.hitfinder_threshold),
result_buffer=hit_bounds_buffer)
# Only view the part of hit_bounds_buffer that contains hits found in this event
# The rest of hit_bounds_buffer contains -1's or stuff from previous pulses
pulse.n_hits_found = n_hits_found
hit_bounds_found = hit_bounds_buffer[:n_hits_found]
if self.always_find_single_hit:
central_bounds = hit_bounds_found
else:
# Extend the boundaries of each hit, to be sure we integrate everything.
# The original bounds are preserved: they are used in clustering
central_bounds = hit_bounds_found.copy()
dsputils.extend_intervals(w, hit_bounds_found, left_extension,
right_extension)
# If no hits were found, this is a noise pulse: update the noise pulse count
if n_hits_found == 0:
event.noise_pulses_in[channel] += 1
# Don't 'continue' to the next pulse! There's stuff left to do!
elif n_hits_found >= self.max_hits_per_pulse:
self.log.debug(
"Pulse %s-%s in channel %s has more than %s hits. "
"This usually indicates a zero-length encoding breakdown after a very large S2. "
"Further hits in this pulse have been ignored." %
(start, stop, channel, self.max_hits_per_pulse))
# Store the found hits in the datastructure
# Convert area, noise_sigma and height from adc counts -> pe
adc_to_pe = dsputils.adc_to_pe(self.config, channel)
noise_sigma_pe = pulse.noise_sigma * adc_to_pe
# If the DAQ pulse was ADC-saturated (clipped), the raw waveform dropped to 0,
# i.e. we went digitizer_reference_baseline above the reference baseline
# i.e. we went digitizer_reference_baseline - pulse.baseline above baseline
# 0.5 is needed to avoid floating-point rounding errors to cause saturation not to be reported
# Somehow happens only when you use simulated data -- apparently np.clip rounds slightly different
saturation_threshold = self.reference_baseline - pulse.baseline - 0.5
build_hits(w, hit_bounds_found, hits_buffer, adc_to_pe, channel,
noise_sigma_pe, dt, start, pulse_i,
saturation_threshold, central_bounds)
hits = hits_buffer[:n_hits_found].copy()
hits_per_pulse.append(hits)
if len(hits_per_pulse):
event.all_hits = np.concatenate(hits_per_pulse)
if not self.always_find_single_hit:
# Remove hits with 0 or negative area (very rare, but possible due to rigid integration bound)
# In always-find-single-hit mode (for PMT calibrations) this is undesirable
event.all_hits = event.all_hits[event.all_hits['area'] > 0]
self.log.debug("Found %d hits in %d pulses" %
(len(event.all_hits), len(event.pulses)))
else:
self.log.warning("Event has no pulses??!")
return event
def transform_event(self, event):
if self.make_diagnostic_plots == 'never':
return event
# Get the pulse-to-hit mapping
# Note this relies on the hits being sorted by found_in_pulse
# (probably shouldn't have rolled our own fake pandas for recarrays...)
self.log.debug("Reconstructing hit/pulse mapping")
hits_in_pulse = dict_group_by(event.all_hits, 'found_in_pulse')
for pulse_i, pulse in enumerate(tqdm(event.pulses, desc='Making hitfinder diagnostic plots')):
# Get the hits that were found in this pulse
hits = hits_in_pulse.get(pulse_i, np.array([], datastructure.Hit.get_dtype()))
# Reconstruct some variables we had in the hitfinder. Some code duplication unfortunately...
# that's the price we pay for having diagnostic plotting cleanly separated from the hitfinder.
start = pulse.left
stop = pulse.right
hit_bounds_found = np.vstack((hits['left'], hits['right'])).T
hit_bounds_found -= start
w = self.reference_baseline - pulse.raw_data.astype(np.float64) - pulse.baseline
channel = pulse.channel
adc_to_pe = dsputils.adc_to_pe(self.config, channel)
noise_sigma_pe = pulse.noise_sigma * adc_to_pe
threshold = pulse.hitfinder_threshold
saturation_threshold = self.reference_baseline - pulse.baseline - 0.5
is_saturated = pulse.maximum >= saturation_threshold
# Do we need to show this pulse? If not: continue
if self.make_diagnostic_plots == 'tricky cases':
# Always show pulse if noise level is very high
if noise_sigma_pe < 0.5:
if len(hit_bounds_found) == 0:
# Show pulse if it nearly went over threshold
if not pulse.maximum > 0.8 * threshold:
continue
else:
# Show pulse if any of its hit nearly didn't go over threshold
if not any([event.all_hits[-(i+1)].height < 1.2 * threshold * adc_to_pe
for i in range(len(hit_bounds_found))]):
continue
elif self.make_diagnostic_plots == 'no hits':
if len(hit_bounds_found) != 0:
continue
elif self.make_diagnostic_plots == 'baseline shifts':
if abs(pulse.baseline_increase) < 10:
continue
elif self.make_diagnostic_plots == 'hits only':
if len(hit_bounds_found) == 0:
continue
elif self.make_diagnostic_plots == 'saturated':
if not is_saturated:
continue
elif self.make_diagnostic_plots == 'negative':
# Select only pulses which had hits whose area and or height were originally negative
# (the hitfinder helpfully capped them at 1e-9, so they're not actually negative...)
if not (len(hits) and (np.any(hits['area'] < 1e-6) or np.any(hits['height'] < 1e-6))):
continue
elif self.make_diagnostic_plots != 'always':
raise ValueError("Invalid make_diagnostic_plots option: %s!" % self.make_diagnostic_plots)
plt.figure(figsize=(14, 10))
data_for_title = (event.event_number, start, stop, channel)
plt.title('Event %s, pulse %d-%d, Channel %d' % data_for_title)
ax1 = plt.gca()
ax2 = ax1.twinx()
ax1.set_position((.1, .1, .6, .85))
ax2.set_position((.1, .1, .6, .85))
ax1.set_xlabel("Sample number (%s ns)" % event.sample_duration)
ax1.set_ylabel("ADC counts above baseline")
ax2.set_ylabel("pe / sample")
# Plot the signal and noise levels
ax1.plot(w, drawstyle='steps-mid', label='Data')
ax1.plot(np.ones_like(w) * threshold, '--', label='Threshold', color='red')
ax1.plot(np.ones_like(w) * pulse.noise_sigma, ':', label='Noise level', color='gray')
ax1.plot(np.ones_like(w) * pulse.minimum, '--', label='Minimum', color='orange')
# Mark the hit ranges & center of gravity point
for hit_i, hit in enumerate(hit_bounds_found):
ax1.axvspan(hit[0] - 0.5, hit[1] + 0.5, color='red', alpha=0.2)
# Make sure the y-scales match
ax2.set_ylim(ax1.get_ylim()[0] * adc_to_pe, ax1.get_ylim()[1] * adc_to_pe)
# Add pulse / hit information
if len(hits) != 0:
largest_hit = hits[np.argmax(hits['area'])]
plt.figtext(0.75, 0.98, dedent("""
Pulse maximum: {pulse.maximum:.5g}
Pulse minimum: {pulse.minimum:.5g}
(both in ADCc above baseline)
Pulse baseline: {pulse.baseline}
(in ADCc above reference baseline)
Baseline increase: {pulse.baseline_increase:.2f}
Gain in this PMT: {gain:.3g}
Noise level: {pulse.noise_sigma:.2f} ADCc
Hitfinder threshold: {pulse.hitfinder_threshold} ADCc
Largest hit info ({left}-{right}):
Area: {hit_area:.5g} pe
Height: {hit_height:.4g} pe
Saturated samples: {hit_n_saturated}
""".format(pulse=pulse,
gain=self.config['gains'][pulse.channel],
left=largest_hit['left']-pulse.left,
right=largest_hit['right']-pulse.left,
hit_area=largest_hit['area'],
hit_height=largest_hit['height'],
hit_n_saturated=largest_hit['n_saturated'])),
fontsize=14, verticalalignment='top')
# Finish the plot, save, close
leg = ax1.legend()
leg.get_frame().set_alpha(0.5)
plt.savefig(os.path.join(self.make_diagnostic_plots_in,
'event%04d_pulse%05d-%05d_ch%03d.png' % data_for_title))
plt.xlim(0, len(pulse.raw_data))
plt.close()
return event
def transform_event(self, event):
# Initialize empty waveforms for each detector
# One with only hits, one with raw data
for postfix in ('', '_raw'):
for detector, chs in self.config['channels_in_detector'].items():
event.sum_waveforms.append(datastructure.SumWaveform(
samples=np.zeros(event.length(), dtype=np.float32),
name=detector + postfix,
channel_list=np.array(list(chs), dtype=np.uint16),
detector=detector
))
# Make dictionary mapping pulse -> non-rejected hits found in pulse
# Assumes hits are still sorted by pulse
event.all_hits = np.sort(event.all_hits, order='found_in_pulse')
hits_per_pulse = recarray_tools.dict_group_by(event.all_hits[True ^ event.all_hits['is_rejected']],
'found_in_pulse')
# Add top and bottom tpc sum waveforms
for q in ('top', 'bottom'):
event.sum_waveforms.append(datastructure.SumWaveform(
samples=np.zeros(event.length(), dtype=np.float32),
name='tpc_%s' % q,
channel_list=np.array(self.config['channels_%s' % q], dtype=np.uint16),
detector='tpc'
))
for pulse_i, pulse in enumerate(event.pulses):
channel = pulse.channel
# Do some initialization only when we switch channel
# The 'current_channel' variable can't be pulled outside the loop, that would hurt performance
# (trust me, try it and time it)
if pulse_i == 0 or channel != current_channel: # noqa
current_channel = channel # noqa
detector = self.detector_by_channel[channel]
adc_to_pe = dsputils.adc_to_pe(self.config, channel)
if detector == 'tpc':
if channel in self.config['channels_top']:
sum_w = event.get_sum_waveform('tpc_top')
else:
sum_w = event.get_sum_waveform('tpc_bottom')
else:
sum_w = event.get_sum_waveform(detector)
# Don't consider dead channels
if self.config['gains'][channel] == 0:
continue
# Get the pulse waveform in pe/bin
baseline_to_subtract = self.config['digitizer_reference_baseline'] - pulse.baseline
w = baseline_to_subtract - pulse.raw_data.astype(np.float64)
w *= adc_to_pe
sum_w_raw = event.get_sum_waveform(detector+'_raw').samples
sum_w_raw[pulse.left:pulse.right+1] += w
hits = hits_per_pulse.get(pulse_i, None)
if hits is None:
continue
w_hits_only = w.copy()
# Obtain an array of same length as w, indicating whether the sample is in a hit or not
mask = np.zeros(len(w), dtype=np.bool)
set_if_in_ranges(mask, hits['left'] - pulse.left, hits['right'] - pulse.left)
w_hits_only[True ^ mask] = 0
sum_w.samples[pulse.left:pulse.right+1] += w_hits_only
# Sum the tpc top and bottom tpc waveforms
event.get_sum_waveform('tpc').samples = event.get_sum_waveform('tpc_top').samples + \
event.get_sum_waveform('tpc_bottom').samples
return event
def WriteHits(self, peak, event, index):
''' Write out 1D histos for this peak '''
# hit_indices = {}
directory = self.outfile.mkdir(peak.type+"_"+str(index))
hists = []
pulses_to_write = {}
hitlist = {}
for hit in peak.hits:
hitdict = {
"found_in_pulse": hit[3],
"area": hit[0],
"channel": hit[2],
"center": hit[1]/10,
"left": hit[7]-1,
"right": hit[10],
"max_index": hit[5],
"height": hit[4]/dsputils.adc_to_pe(self.config, hit[2],
use_reference_gain=True)
}
if hitdict['channel'] not in hitlist:
hitlist[hitdict['channel']] = []
pulses_to_write[hitdict['channel']] = []
hitlist[hitdict['channel']].append(hitdict)
if hitdict['found_in_pulse'] not in pulses_to_write[hitdict['channel']]:
pulses_to_write[hitdict['channel']].append(hitdict['found_in_pulse'])
# Now we should have pulses_to_write with which
# pulses to plot per channel and
# hitlist with a list of all hit properties to add
for channel, pulselist in pulses_to_write.items():
leftbound = -1
rightbound = -1
# Needs an initial scan to find histogram range
for pulseid in pulselist:
if leftbound == -1 or event.pulses[pulseid].left < leftbound:
leftbound = event.pulses[pulseid].left
if rightbound == -1 or event.pulses[pulseid].right > rightbound:
rightbound = event.pulses[pulseid].right
# Make and book the histo. Put into hists so doesn't get overwritten
histname = "%s_%i_channel_%i" % (peak.type, index, channel)
histtitle = "Channel %i in %s[%i]" % (channel, peak.type, index)
c = ROOT.TCanvas(histname, "")
h = ROOT.TH1F(histname, histtitle, int(rightbound-leftbound),
float(leftbound), float(rightbound))
# Now put the bin values in the histogram
for pulseid in pulselist:
pulse = event.pulses[pulseid]
w = (self.config['digitizer_reference_baseline'] + pulse.baseline -
pulse.raw_data.astype(np.float64))
for i, sample in enumerate(w):
h.SetBinContent(int(i+pulse.left-leftbound), sample)
h.SetStats(0)
h.GetXaxis().SetTitle("Time [samples]")
h.GetYaxis().SetTitleOffset(0.8)
h.GetYaxis().SetTitleSize(0.05)
h.GetXaxis().SetTitleOffset(0.8)
h.GetXaxis().SetTitleSize(0.05)
h.GetYaxis().SetTitle("ADC Reading (baseline corrected)")
c.cd()
h.Draw()
plist = {"x": [], "y": []}
for i, hitdict in enumerate(hitlist[channel]):
baseline = ROOT.TLine(hitdict['left'], pulse.baseline,
hitdict['right'], pulse.baseline)
leftline = ROOT.TLine(hitdict['left'], 0, hitdict['left'], hitdict['height'])
rightline = ROOT.TLine(hitdict['right'], 0, hitdict['right'], hitdict['height'])
plist['x'].append(hitdict['center'])
plist['y'].append(hitdict['height'])
leftline.SetLineStyle(2)
rightline.SetLineStyle(2)
leftline.SetLineColor(2)
rightline.SetLineColor(2)
baseline.SetLineStyle(2)
baseline.SetLineColor(4)
baseline.Draw("same")
leftline.Draw("same")
rightline.Draw("same")
label = "hit " + str(i) + "({:.2f} p.e.)".format(hitdict['area'])
text = ROOT.TText(hitdict['center'], hitdict['height'], label)
text.SetTextColor(2)
text.SetTextSize(0.03)
text.Draw("same")
c.Update()
hists.append({"lline": leftline, "text": text,
"rline": rightline, "bline": baseline})
c.cd()
polymarker = ROOT.TPolyMarker()
polymarker.SetMarkerStyle(23)
polymarker.SetMarkerColor(2)
polymarker.SetMarkerSize(1.1)
polymarker.SetPolyMarker(len(plist['x']), np.array(plist['x']),
np.array(plist['y']))
polymarker.Draw("same")
c.Update()
hists.append({"poly": polymarker, "hist": h, "c": c})
directory.cd()
c.Write()
return hists
def show_time_range(st, run_id, t0, dt=10):
from functools import partial
import numpy as np
import pandas as pd
import holoviews as hv
from holoviews.operation.datashader import datashade, dynspread
hv.extension('bokeh')
import strax
import gc
# Somebody thought it was a good idea to call gc.collect explicitly somewhere in holoviews
# This makes dynamic PMT maps super slow
# Until I trace the offender:
gc.collect = lambda *args, **kwargs: None
# Custom wheel zoom tool that only zooms in time
from bokeh.models import WheelZoomTool
time_zoom = WheelZoomTool(dimensions='width')
# Get ADC->pe multiplicative conversion factor
from pax.configuration import load_configuration
from pax.dsputils import adc_to_pe
pax_config = load_configuration('XENON1T')["DEFAULT"]
to_pe = np.array(
[adc_to_pe(pax_config, ch) for ch in range(pax_config['n_channels'])])
tpc_r = pax_config['tpc_radius']
# Get locations of PMTs
r = []
for q in pax_config['pmts']:
r.append(
dict(x=q['position']['x'],
y=q['position']['y'],
i=q['pmt_position'],
array=q.get('array', 'other')))
f = 1.08
pmt_locs = pd.DataFrame(r)
records = st.get_array(run_id,
'raw_records',
time_range=(t0, t0 + int(1e10)))
# TOOD: don't reprocess, just load...
hits = strax.find_hits(records)
peaks = strax.find_peaks(hits,
to_pe,
gap_threshold=300,
min_hits=3,
result_dtype=strax.peak_dtype(n_channels=260))
strax.sum_waveform(peaks, records, to_pe)
# Integral in pe
areas = records['data'].sum(axis=1) * to_pe[records['channel']]
def normalize_time(t):
return (t - records[0]['time']) / 1e9
# Create dataframe with record metadata
df = pd.DataFrame(
dict(area=areas,
time=normalize_time(records['time']),
channel=records['channel']))
# Convert to holoviews Points
points = hv.Points(
df,
kdims=[
hv.Dimension('time', label='Time', unit='sec'),
hv.Dimension('channel', label='PMT number', range=(0, 260))
],
vdims=[
hv.Dimension(
'area',
label='Area',
unit='pe',
# range=(0, 1000)
)
])
def pmt_map(t_0, t_1, array='top', **kwargs):
# Compute the PMT pattern (fast)
ps = points[(t_0 <= points['time']) & (points['time'] < t_1)]
areas = np.bincount(ps['channel'],
weights=ps['area'],
minlength=len(pmt_locs))
# Which PMTs should we include?
pmt_mask = pmt_locs['array'] == array
d = pmt_locs[pmt_mask].copy()
d['area'] = areas[pmt_mask]
# Convert to holoviews points
d = hv.Dataset(d,
kdims=[
hv.Dimension('x',
unit='cm',
range=(-tpc_r * f, tpc_r * f)),
hv.Dimension('y',
unit='cm',
range=(-tpc_r * f, tpc_r * f)),
hv.Dimension('i', label='PMT number'),
hv.Dimension('area', label='Area', unit='PE')
])
return d.to(hv.Points,
vdims=['area', 'i'],
group='PMTPattern',
label=array.capitalize(),
**kwargs).opts(plot=dict(color_index=2,
tools=['hover'],
show_grid=False),
style=dict(size=17, cmap='magma'))
def pmt_map_range(x_range, array='top', **kwargs):
# For use in dynamicmap with streams
if x_range is None:
x_range = (0, 0)
return pmt_map(x_range[0], x_range[1], array=array, **kwargs)
xrange_stream = hv.streams.RangeX(source=points)
# TODO: weigh by area
def channel_map():
return dynspread(
datashade(
points, y_range=(0, 260),
streams=[xrange_stream])).opts(plot=dict(
width=600,
tools=[time_zoom, 'xpan'],
default_tools=['save', 'pan', 'box_zoom', 'save', 'reset'],
show_grid=False))
def plot_peak(p):
# It's better to plot amplitude /time than per bin, since
# sampling times are now variable
y = p['data'][:p['length']] / p['dt']
t_edges = np.arange(p['length'] + 1, dtype=np.int64)
t_edges = t_edges * p['dt'] + p['time']
t_edges = normalize_time(t_edges)
# Correct step plotting from Knut
t_ = np.zeros(2 * len(y))
y_ = np.zeros(2 * len(y))
t_[0::2] = t_edges[0:-1]
t_[1::2] = t_edges[1::]
y_[0::2] = y
y_[1::2] = y
c = hv.Curve(dict(time=t_, amplitude=y_),
kdims=points.kdims[0],
vdims=hv.Dimension('amplitude',
label='Amplitude',
unit='PE/ns'),
group='PeakSumWaveform')
return c.opts(
plot=dict( # interpolation='steps-mid',
# default_tools=['save', 'pan', 'box_zoom', 'save', 'reset'],
# tools=[time_zoom, 'xpan'],
width=600,
shared_axes=False,
show_grid=True),
style=dict(color='b')
# norm=dict(framewise=True)
)
def peaks_in(t_0, t_1):
return peaks[(normalize_time(peaks['time'] +
peaks['length'] * peaks['dt']) > t_0)
& (normalize_time(peaks['time']) < t_1)]
def plot_peaks(t_0, t_1, n_max=10):
# Find peaks in this range
ps = peaks_in(t_0, t_1)
# Show only the largest n_max peaks
if len(ps) > n_max:
areas = ps['area']
max_area = np.sort(areas)[-n_max]
ps = ps[areas >= max_area]
return hv.Overlay(items=[plot_peak(p) for p in ps])
def plot_peak_range(x_range, **kwargs):
# For use in dynamicmap with streams
if x_range is None:
x_range = (0, 10)
return plot_peaks(x_range[0], x_range[1], **kwargs)
top_map = hv.DynamicMap(partial(pmt_map_range, array='top'),
streams=[xrange_stream])
bot_map = hv.DynamicMap(partial(pmt_map_range, array='bottom'),
streams=[xrange_stream])
waveform = hv.DynamicMap(plot_peak_range, streams=[xrange_stream])
layout = waveform + top_map + channel_map() + bot_map
return layout.cols(2)
def plot_event(self, event):
dt = self.config['sample_duration']
# Configurable limits
channels_start = self.config.get('channel_start', min(self.config['channels_in_detector']['tpc']))
channels_end = self.config.get('channel_start', max(self.config['channels_in_detector']['tpc']))
t_start = self.config.get('t_start', 0 * units.us)
t_end = self.config.get('t_end', event.duration())
fig = plt.figure(figsize=(self.size_multiplier * 4, self.size_multiplier * 2))
ax = fig.gca(projection='3d')
global_max_amplitude = 0
for pulse in event.pulses:
# Take only channels in the range we want to plot
if not channels_start <= pulse.channel <= channels_end:
continue
# Take only pulses that fall (at least partially) in the time window we want to plot
if pulse.right * dt < t_start or pulse.left * dt > t_end:
continue
w = self.config['digitizer_reference_baseline'] + pulse.baseline - pulse.raw_data.astype(np.float64)
w *= dsputils.adc_to_pe(self.config, pulse.channel, use_reference_gain=True)
if self.config['log_scale']:
# This will still give nan's if waveform drops below 1 pe_nominal / bin...
# TODO: So... will it crash? or just fall outside range?
w = np.log10(1 + w)
# We need to keep track of this, apparently gca can't scale itself?
global_max_amplitude = max(np.max(w), global_max_amplitude)
ax.plot(
np.linspace(pulse.left, pulse.right, pulse.length) * dt / units.us,
pulse.channel * np.ones(pulse.length),
zs=w,
zdir='z',
label=str(pulse.channel)
)
# Plot the sum waveform
w = event.get_sum_waveform('tpc').samples[int(t_start / dt):int(t_end / dt) + 1]
ax.plot(
np.linspace(t_start, t_end, len(w)) / units.us,
(channels_end + 1) * np.ones(len(w)),
zs=w * global_max_amplitude / np.max(w),
zdir='z',
label='Tpc'
)
ax.set_xlabel('Time [$\mu$s]')
ax.set_xlim3d(t_start / units.us, t_end / units.us)
ax.set_ylabel('Channel number')
ax.set_ylim3d(channels_start, channels_end)
zlabel = 'Pulse height [pe_nominal / %d ns]' % (self.config['sample_duration'] / units.ns)
if self.config['log_scale']:
zlabel = 'Log10 1 + ' + zlabel
ax.set_zlabel(zlabel)
ax.set_zlim3d(0, global_max_amplitude)
def split_peak(self, peak, split_points):
"""Yields new peaks split from peak at split_points = sample indices within peak
Samples at the split points will fall to the right (so if we split [0, 5] on 2, you get [0, 1] and [2, 5]).
Hits that straddle a split point are themselves split into two hits: peak.hits is updated.
"""
# First, split hits that straddle the split points
# Hits may have to be split several times; for each split point we modify the 'hits' list, splitting only
# the hits we need.
hits = peak.hits
for x in split_points:
x += peak.left # Convert to index in event
# Select hits that must be split: start before x and end after it.
selection = (hits['left'] <= x) & (hits['right'] > x)
hits_to_split = hits[selection]
# new_hits will be a list of hit arrays, which we concatenate later to make the new 'hits' list
# Start with the hits that don't have to be split: we definitely want to retain those!
new_hits = [hits[True ^ selection]]
for h in hits_to_split:
pulse_i = h['found_in_pulse']
pulse = self.event.pulses[pulse_i]
# Get the pulse waveform in ADC counts above baseline (because it's what build_hits expect)
baseline_to_subtract = self.config['digitizer_reference_baseline'] - pulse.baseline
w = baseline_to_subtract - pulse.raw_data.astype(np.float64)
# Use the hitfinder's build_hits to compute the properties of these hits
# Damn this is ugly... but at least we don't have duplicate property computation code
hits_buffer = np.zeros(2, dtype=datastructure.Hit.get_dtype())
adc_to_pe = dsputils.adc_to_pe(self.config, h['channel'])
hit_bounds = np.array([[h['left'], x], [x+1, h['right']]], dtype=np.int64)
hit_bounds -= pulse.left # build_hits expects hit bounds relative to pulse start
build_hits(w,
hit_bounds=hit_bounds,
hits_buffer=hits_buffer,
adc_to_pe=adc_to_pe,
channel=h['channel'],
noise_sigma_pe=pulse.noise_sigma * adc_to_pe,
dt=self.config['sample_duration'],
start=pulse.left,
pulse_i=pulse_i,
saturation_threshold=self.config['digitizer_reference_baseline'] - pulse.baseline - 0.5,
central_bounds=hit_bounds) # TODO: Recompute central bounds in an intelligent way...
# Remove hits with 0 or negative area (very rare, but possible due to rigid integration bound)
hits_buffer = hits_buffer[hits_buffer['area'] > 0]
new_hits.append(hits_buffer)
# Now remake the hits list, then go on to the next peak.
hits = np.concatenate(new_hits)
# Next, split the peaks, sorting hits to the right peak by their maximum index.
# Iterate over left, right bounds of the new peaks
boundaries = list(zip([0] + [y+1 for y in split_points], split_points + [float('inf')]))
for l, r in boundaries:
# Convert to index in event
l += peak.left
r += peak.left
# Select hits which have their maximum within this peak bounds
# The last new peak must also contain hits at the right bound (though this is unlikely to happen)
hs = hits[(hits['index_of_maximum'] >= l) &
(hits['index_of_maximum'] <= r)]
if not len(hs):
# Hits have probably been removed by area > 0 condition
self.log.info("Localminimumclustering requested creation of peak %s-%s without hits. "
"This is a possible outcome if there are large oscillations in one channel, "
"but it should be very rare." % (l, r))
continue
r = r if r < float('inf') else peak.right
if not len(hs):
raise RuntimeError("Attempt to create a peak without hits in LocalMinimumClustering!")
yield self.build_peak(hits=hs, detector=peak.detector, left=l, right=r)
def split_peak(self, peak, split_points):
"""Yields new peaks split from peak at split_points = sample indices within peak
Samples at the split points will fall to the right (so if we split [0, 5] on 2, you get [0, 1] and [2, 5]).
Hits that straddle a split point are themselves split into two hits: peak.hits is updated.
"""
# First, split hits that straddle the split points
# Hits may have to be split several times; for each split point we modify the 'hits' list, splitting only
# the hits we need.
hits = peak.hits
for x in split_points:
x += peak.left # Convert to index in event
# Select hits that must be split: start before x and end after it.
selection = (hits['left'] <= x) & (hits['right'] > x)
hits_to_split = hits[selection]
# new_hits will be a list of hit arrays, which we concatenate later to make the new 'hits' list
# Start with the hits that don't have to be split: we definitely want to retain those!
new_hits = [hits[True ^ selection]]
for h in hits_to_split:
pulse_i = h['found_in_pulse']
pulse = self.event.pulses[pulse_i]
# Get the pulse waveform in ADC counts above baseline (because it's what build_hits expect)
baseline_to_subtract = self.config[
'digitizer_reference_baseline'] - pulse.baseline
w = baseline_to_subtract - pulse.raw_data.astype(np.float64)
# Use the hitfinder's build_hits to compute the properties of these hits
# Damn this is ugly... but at least we don't have duplicate property computation code
hits_buffer = np.zeros(2, dtype=datastructure.Hit.get_dtype())
adc_to_pe = dsputils.adc_to_pe(self.config, h['channel'])
hit_bounds = np.array([[h['left'], x], [x + 1, h['right']]],
dtype=np.int64)
hit_bounds -= pulse.left # build_hits expects hit bounds relative to pulse start
build_hits(
w,
hit_bounds=hit_bounds,
hits_buffer=hits_buffer,
adc_to_pe=adc_to_pe,
channel=h['channel'],
noise_sigma_pe=pulse.noise_sigma * adc_to_pe,
dt=self.config['sample_duration'],
start=pulse.left,
pulse_i=pulse_i,
saturation_threshold=self.
config['digitizer_reference_baseline'] - pulse.baseline -
0.5,
central_bounds=hit_bounds
) # TODO: Recompute central bounds in an intelligent way...
# Remove hits with 0 or negative area (very rare, but possible due to rigid integration bound)
hits_buffer = hits_buffer[hits_buffer['area'] > 0]
new_hits.append(hits_buffer)
# Now remake the hits list, then go on to the next peak.
hits = np.concatenate(new_hits)
# Next, split the peaks, sorting hits to the right peak by their maximum index.
# Iterate over left, right bounds of the new peaks
boundaries = list(
zip([0] + [y + 1 for y in split_points],
split_points + [float('inf')]))
for left, right in boundaries:
# Convert to index in event
left += peak.left
right += peak.left
# Select hits which have their maximum within this peak bounds
# The last new peak must also contain hits at the right bound (though this is unlikely to happen)
hs = hits[(hits['index_of_maximum'] >= left)
& (hits['index_of_maximum'] <= right)]
if not len(hs):
# Hits have probably been removed by area > 0 condition
self.log.info(
"Localminimumclustering requested creation of peak %s-%s without hits. "
"This is a possible outcome if there are large oscillations in one channel, "
"but it should be very rare." % (left, right))
continue
right = right if right < float('inf') else peak.right
if not len(hs):
raise RuntimeError(
"Attempt to create a peak without hits in LocalMinimumClustering!"
)
yield self.build_peak(hits=hs,
detector=peak.detector,
left=left,
right=right)