def make_template(data, ignore=None, length=2000, fig=None):
"""
Make a template waveform for the matched filter.
Parameters
----------
data : array (nevents, 2, 15001)
As returned by readwav.readwav().
ignore : bool array (nevents,), optional
Flag events to be ignored.
length : int
Number of samples of the waveform.
fig : matplotlib figure, optional
If given, plot the waveform.
Return
------
waveform : array (length,)
The waveform. It is normalized to unit sum.
"""
if ignore is None:
ignore = np.zeros(len(data), bool)
# Run a moving average filter to find and separate the signals by number of
# photoelectrons.
trigger, baseline, value = integrate.filter(data, bslen=8000, length_ma=1470, delta_ma=1530)
corr_value = baseline - value[:, 0]
snr, center, width = single_filter_analysis(corr_value[~ignore], return_full=True)
assert snr > 15
assert len(center) > 2
# Select the data corresponding to 1 photoelectron and subtract the
# baseline.
lower = (center[0] + center[1]) / 2
upper = (center[1] + center[2]) / 2
selection = (lower < corr_value) & (corr_value < upper) & ~ignore
t = int(np.median(trigger))
data1pe = data[selection, 0, t:t + length] - baseline[selection].reshape(-1, 1)
# Compute the waveform as the median of the signals.
waveform, bottom, top = np.quantile(data1pe, [0.5, 0.25, 0.75], axis=0)
# waveform = np.mean(data1pe, axis=0)
# std = np.std(data1pe, axis=0)
# bottom = waveform - std
# top = waveform + std
if fig is not None:
ax = fig.subplots(1, 1)
ax.fill_between(np.arange(length), bottom, top, facecolor='lightgray', label='25%-75% quantiles')
ax.plot(waveform, 'k-', label='median')
ax.grid()
ax.legend(loc='best')
ax.set_xlabel('Sample number')
ax.set_ylabel('ADC value')
ax.set_title('Template for matched filter (unnormalized)')
return waveform / np.sum(waveform)
def fingerplot(delta=0, bslen=8000):
"""
Make a fingerplot with the matched filter.
Parameters
----------
delta : int
The offset from the trigger minus the waveform length.
bslen : int
Number of samples used for baseline computation.
"""
fig1 = plt.figure('fingersnrmf-fingerplot-1', figsize=[7.27, 5.73])
fig2 = plt.figure('fingersnrmf-fingerplot-2', figsize=[6.4, 4.8])
fig1.clf()
fig2.clf()
trigger, baseline, value = integrate.filter(data, bslen=bslen, delta_mf=len(waveform) + delta, waveform_mf=waveform)
value = value[:, 0]
corr_value = (baseline - value)[~ignore]
snr = single_filter_analysis(corr_value, fig1, fig2)
print(f'snr = {snr:.2f}')
fig1.tight_layout()
fig2.tight_layout()
fig1.show()
fig2.show()
def snrseries(deltamin=-50, deltamax=50, ndelta=101, bslen=8000):
"""
Compute SNR as a function of the offset from the trigger where the filter
is evaluated ("delta").
Parameters
----------
deltamin, deltamax, ndelta: int
The delta values where the SNR is computed is a range of ndelta values
from deltamin to deltamax. The range is specified relative to the
number of samples of the waveform, i.e. delta=0 -> delta=len(waveform).
bslen : int
The number of samples used for the baseline.
Returns
-------
delta : array (ndelta,)
Values of delta.
snr : array (ndelta,)
The SNR for each delta.
"""
delta = np.rint(np.linspace(deltamin, deltamax, ndelta)) + len(waveform)
start, baseline, value = integrate.filter(data, bslen=bslen, delta_mf=delta, waveform_mf=waveform)
snr = np.empty(len(delta))
for i in tqdm.tqdm(range(len(snr))):
val = value[:, i]
corr_value = (baseline - val)[~ignore]
snr[i] = single_filter_analysis(corr_value)
output = delta, snr
snrplot(*output)
return output
def fun(delta, tau, kind, waveform, start_mf):
try:
if kind == 'exp':
start, baseline, value = integrate.filter(self.data,
bslen,
delta_exp=delta,
tau_exp=tau)
elif kind == 'ma':
start, baseline, value = integrate.filter(self.data,
bslen,
delta_ma=delta,
length_ma=tau)
elif kind == 'mf':
start, baseline, value = integrate.filter(
self.data,
bslen,
delta_mf=delta,
length_mf=tau,
waveform_mf=waveform,
start_mf=start_mf)
else:
raise KeyError(kind)
except ZeroDivisionError:
return 0
corr_value = (baseline - value[:, 0])[~self.ignore]
snr = single_filter_analysis(corr_value)
return -snr
def make_template(data,
ignore=None,
length=2000,
noisecorr=False,
fig=None,
figcov=None,
norm=True):
"""
Make a template waveform for the matched filter.
Parameters
----------
data : array (nevents, 2, 15001)
As returned by readwav.readwav().
ignore : bool array (nevents,), optional
Flag events to be ignored.
length : int
Number of samples of the waveform.
noisecorr : bool
If True, optimize the filter for the noise spectrum.
fig : matplotlib figure, optional
If given, plot the waveform.
figcov : matplotlib figure, optional
If given, plot the covariance matrix of the noise as a bitmap.
norm : bool
If True, normalize the output to unit sum, so that applying it behaves
like a weighted mean.
Return
------
waveform : array (length,)
The waveform.
"""
if ignore is None:
ignore = np.zeros(len(data), bool)
# Run a moving average filter to find and separate the signals by number of
# photoelectrons.
trigger, baseline, value = integrate.filter(data,
bslen=8000,
length_ma=1470,
delta_ma=1530)
corr_value = baseline - value[:, 0]
snr, center, width = single_filter_analysis(corr_value[~ignore],
return_full=True)
assert snr > 15
assert len(center) > 2
# Select the data corresponding to 1 photoelectron and subtract the
# baseline.
lower = (center[0] + center[1]) / 2
upper = (center[1] + center[2]) / 2
selection = (lower < corr_value) & (corr_value < upper) & ~ignore
t = int(np.median(trigger))
data1pe = data[selection, 0, t:t + length] - baseline[selection].reshape(
-1, 1)
# Compute the waveform as the median of the signals.
waveform = np.median(data1pe, axis=0)
# waveform = np.mean(data1pe, axis=0)
if noisecorr:
# Select baseline data and compute covariance matrix over a slice with
# the same length of the template.
bsend = t - 100
N = 2 * (length + length % 2)
databs = data[~ignore, 0, bsend - N:bsend]
cov = np.cov(databs, rowvar=False)
cov = toeplitze(cov)
s = slice(N // 4, N // 4 + length)
cov = cov[s, s]
# use cov(fweights=~ignore) to avoid using ~ignore
# Correct the waveform.
wnocov = waveform
waveform = linalg.solve(cov, waveform, assume_a='pos')
waveform *= linalg.norm(cov) / len(waveform)
if fig is not None:
axs = fig.subplots(2, 1)
ax = axs[0]
if noisecorr:
ax.plot(wnocov / np.max(np.abs(wnocov)),
label='assuming white noise')
ax.plot(waveform / np.max(np.abs(waveform)),
label='corrected for actual noise',
zorder=-1)
else:
# std = np.std(data1pe, axis=0)
# bottom = waveform - std
# top = waveform + std
bottom, top = np.quantile(data1pe, [0.25, 0.75], axis=0)
ax.fill_between(np.arange(length),
bottom,
top,
facecolor='lightgray',
label='25%-75% quantiles')
ax.plot(waveform, 'k-', label='median')
ax.set_ylabel('ADC value')
ax.grid()
ax.legend(loc='best')
ax.set_xlabel('Sample number')
ax.set_title('Template for matched filter')
ax = axs[1]
f, s = signal.periodogram(wnocov if noisecorr else waveform,
window='hann')
ax.plot(f[1:], np.sqrt(s[1:]), label='spectrum of template')
f, ss = signal.periodogram(data1pe, axis=-1)
s = np.median(ss, axis=0)
ax.plot(f[1:], np.sqrt(s[1:]), label='spectrum of template sources')
ax.set_ylabel('Spectral density [GHz$^{-1/2}$]')
ax.set_xlabel('Frequency [GHz]')
ax.grid()
ax.set_yscale('log')
ax.legend(loc='best')
if noisecorr and figcov is not None:
ax = figcov.subplots(1, 1)
m = np.max(np.abs(cov))
ax.imshow(cov, vmin=-m, vmax=m, cmap='PiYG')
ax.set_title('Noise covariance matrix')
ax.set_xlabel('Sample number [ns]')
ax.set_ylabel('Sample number [ns]')
if norm:
waveform /= np.sum(waveform)
return waveform
rep = 1
filename = 'darksidehd/nuvhd_lf_3x_tile57_77K_64V_6VoV_1.wav'
###########################
data = readwav.readwav(filename, mmap=False)
mask = ~readwav.spurious_signals(data)
# Run a moving average filter to find and separate the signals by
# number of photoelectrons.
trigger, baseline, value = integrate.filter(data,
bslen=8000,
length_ma=1470,
delta_ma=1530)
corr_value = baseline - value[:, 0]
snr, center, width = single_filter_analysis.single_filter_analysis(
corr_value[mask], return_full=True)
assert snr > 15
assert len(center) > 2
# Select the data corresponding to a different number of photoelectrons.
nphotons = np.sort(nphotons)
datanph = []
for nph in nphotons:
lower = (center[nph - 1] + center[nph]) / 2
upper = (center[nph] + center[nph + 1]) / 2
selection = (lower < corr_value) & (corr_value < upper) & mask
indices = np.flatnonzero(selection)[:rep]
indices0 = indices[:, None]
indices2 = trigger[indices, None] + np.arange(-leftmargin, length)
datanph.append(data[indices0, 0, indices2])
import numpy as np
from matplotlib import pyplot as plt
import readwav
import single_filter_analysis
import integrate
import figlatex
filename = 'darksidehd/nuvhd_lf_3x_tile57_77K_64V_6VoV_1.wav'
length = 128
###########################
data = readwav.readwav(filename, mmap=False)
mask = ~readwav.spurious_signals(data)
trigger, baseline, value = integrate.filter(data,
bslen=8000,
length_ma=length,
delta_ma=length)
corr_value = baseline - value[:, 0]
fig = plt.figure(num='figfingerplot', clear=True, figsize=[6.4, 3.5])
single_filter_analysis.single_filter_analysis(corr_value[mask], fig)
fig.tight_layout()
fig.show()
figlatex.save(fig)
filename = sys.argv[1]
length = int(sys.argv[2])
maxevents = int(sys.argv[3])
except:
pass
data = readwav.readwav(filename, mmap=False, maxevents=maxevents)
mask = ~np.any(data[:, 0, :8000] < 700, axis=-1)
if data.shape[1] == 2:
trigger, baseline, value = integrate.filter(data, bslen=8000, length_ma=length, delta_ma=length)
value = value[:, 0]
else:
baseline = np.median(data[:, 0, :8000], axis=-1)
start = 8969 - 21
value = np.mean(data[:, 0, start:start + length], axis=-1)
corr_value = baseline - value
fig = plt.figure(num='fingerplot', clear=True)
snr, _, _ = single_filter_analysis.single_filter_analysis(corr_value[mask], fig, return_full=True)
print(f'SNR = {snr:.3g}')
ax = fig.get_axes()[0]
ax.set_title(os.path.split(filename)[1])
textbox.textbox(ax, f"""\
first {len(data)} events
ignored {np.count_nonzero(~mask)} events
movavg {length} ns""", fontsize='small', loc='center right')
fig.tight_layout()
fig.show()
def fingerplot(self, tau, delta, kind='ma', bslen=8000):
"""
Make a fingerplot with a specific filter and print the SNR.
Parameters
----------
tau : int
Length parameter of the filter.
delta : int
Offset from the trigger where the filter is evaluated.
kind : str
One of 'ma' = moving average, 'exp' = exponential moving average,
'mf' = matched filter, 'mfn' = matched filter with noise correction.
bslen : int
Number of samples used for the baseline.
Return
------
fig1, fig2 : matplotlib figures
"""
if kind == 'ma':
start, baseline, value = integrate.filter(self.data,
bslen,
delta_ma=delta,
length_ma=tau)
elif kind == 'exp':
start, baseline, value = integrate.filter(self.data,
bslen,
delta_exp=delta,
tau_exp=tau)
elif kind in ('mf', 'mfn'):
w0, offset = self.template.matched_filter_template(
self.template.template_length, timebase=1, aligned='trigger')
assert offset == 0, offset
start_mf = integrate.make_start_mf(w0, tau)
if kind == 'mfn':
waveform = make_template.make_template(self.data,
self.ignore,
tau + start_mf[0],
noisecorr=True)
else:
waveform = w0
start, baseline, value = integrate.filter(self.data,
bslen,
delta_mf=delta,
waveform_mf=waveform,
length_mf=tau,
start_mf=start_mf)
else:
raise KeyError(kind)
corr_value = (baseline - value[:, 0])[~self.ignore]
fig1 = plt.figure('fingersnr-fingerplot-1', figsize=[7.27, 5.73])
fig2 = plt.figure('fingersnr-fingerplot-2', figsize=[6.4, 4.8])
fig1.clf()
fig2.clf()
snr = single_filter_analysis(corr_value, fig1, fig2)
print(f'snr = {snr:.2f}')
fig1.tight_layout()
fig2.tight_layout()
fig1.show()
fig2.show()
return fig1, fig2
def snrseries(self,
tau=_default_tau,
ndelta=_default_ndelta,
bslen=8000,
plot=True):
"""
Compute SNR as a function of tau and delta. Make a plot and return the
results.
Parameters
----------
tau : array (ntau,)
Length parameter of the filters.
ndelta : int
Number of values of offset from trigger explored in a hardcoded range.
bslen : int
The number of samples used for the baseline.
plot : bool
If False, do not plot. The plot can be done separately by calling
snrplot().
Returns
-------
tau : array (ntau,)
Values of the filter scale parameter.
delta_ma : array (ntau, ndelta)
Values of the offset for the moving average for each tau.
delta_exp : array (ntau, ndelta)
Values of the offset for the exponential moving average for each tau.
delta_mf : array (ntau, ndelta)
Values of the offset for the matched filter for each tau.
waveform : array (max(tau),)
Template used for the matched filter.
snr : array (3, ntau, ndelta)
The SNR for (moving average, exponential moving average, matched
filter), and for each length parameter (tau) and offset from trigger
(delta).
"""
# Generate delta ranges.
ntau = len(tau)
tau, delta_ma, delta_exp, delta_mf = self.make_tau_delta(tau,
ndelta,
flat=True)
print('make template for matched filter...')
# w0 = make_template.make_template(self.data, self.ignore, np.max(tau) + 200, noisecorr=False)
w0, offset = self.template.matched_filter_template(
self.template.template_length, timebase=1, aligned='trigger')
assert offset == 0, offset
start_mf = integrate.make_start_mf(w0, tau)
# waveform = make_template.make_template(self.data, self.ignore, np.max(tau + start_mf), noisecorr=True)
waveform = w0
print('computing filters...')
start, baseline, vma, vexp, vmf = integrate.filter(
self.data, bslen, delta_ma, tau, delta_exp, tau, delta_mf,
waveform, tau, start_mf)
snr = np.empty((3, len(tau)))
print('analysing filter output...')
for i in tqdm.tqdm(range(snr.shape[1])):
for j, value in enumerate([vma, vexp, vmf]):
value = value[:, i]
corr_value = (baseline - value)[~self.ignore]
snr[j, i] = single_filter_analysis(corr_value)
# Reshape arrays, make plot and return.
output = (tau.reshape(ntau, ndelta)[:, 0], )
for x in [delta_ma, delta_exp, delta_mf]:
output += (x.reshape(ntau, ndelta), )
output += (waveform, snr.reshape(-1, ntau, ndelta))
if plot:
self.snrplot(*output)
return output
def from_lngs(cls, data, length, mask=None, trigger=None, fig=None):
"""
Compute a template from 1 p.e. signals in an LNGS wav.
Parameters
----------
data : array (nevents, nchannels, 15001)
Wav data as read by readwav.readwav(). If it has two channels, the
second channel is the trigger. If there is only one channel,
specify the trigger position with `trigger`.
length : int
Number of samples of the template (@ 1 GSa/s), starting from the
beginning of the trigger impulse.
mask : bool array (nevents,), optional
Mask for the `data` array.
trigger : int, optional
Position of the trigger start in the events. If not specified,
the trigger is read from the second channel of `data`. If specified,
it supersedes `data` even with two channels.
fig : matplotlib figure, optional
If provided, the fingerplot used to select 1 pe pulses is plotted
here.
Return
------
self : Template
A template object.
"""
self = cls.__new__(cls)
if mask is None:
mask = np.ones(len(data), bool)
# Find the trigger.
if trigger is None:
hastrigger = True
trigger = firstbelowthreshold.firstbelowthreshold(data[:, 1], 600)
else:
hastrigger = False
trigger = np.full(len(data), trigger)
# Find spurious signals.
baseline_zone = data[:, 0, :np.min(trigger) - 1000]
spurious = firstbelowthreshold.firstbelowthreshold(baseline_zone, 700) >= 0
mask = mask & ~spurious
# Count photoelectrons using the average.
baseline = np.median(baseline_zone, axis=-1)
value = meanat(data[:, 0], trigger - 21, 1500)
corr_value = baseline - value
snr, center, width = single_filter_analysis(corr_value[mask], return_full=True, fig1=fig)
assert len(center) > 2, len(center)
if snr == 0:
print(f'warning: 0 and 1 pe peaks may have unexpected position')
oursnr = center[1] / width[0]
minsnr = 5
assert oursnr >= minsnr, f'SNR = {oursnr:.3g} < {minsnr}'
# Select the data corresponding to 1 photoelectron.
lower = (center[0] + center[1]) / 2
upper = (center[1] + center[2]) / 2
selection = (lower < corr_value) & (corr_value < upper) & mask
if fig is not None:
ax, = fig.get_axes()
ax.axvspan(lower, upper, color='#f55', zorder=-1, label='selection for template')
ax.legend(loc='upper right')
# Compute the waveform as the mean of the signals.
mtrig = np.full(len(trigger), np.median(trigger))
template = vecmeanat(data[:, 0], baseline, selection, mtrig, length)
# Do a 2-sample moving average to remove the nyquist noise.
tcs = np.pad(np.cumsum(template), (1, 0))
filttempl = (tcs[2:] - tcs[:-2]) / 2
# Redo the template aligning with the cross correlation filter.
delta = 100
t = trigger[0]
filtered = signal.fftconvolve(data[selection, 0, t - delta:t + delta + length - 1], -filttempl[None, ::-1], axes=-1, mode='valid')
indices = np.flatnonzero(selection)
assert filtered.shape == (len(indices), 2 * delta + 1)
idx = argminrelmin.argminrelmin(filtered, axis=-1)
selection[indices] &= idx >= 0
start = np.zeros(len(data), int)
start[indices] = t - delta + idx
template_aligned = vecmeanat(data[:, 0], baseline, selection, start, length)
# Redo the template aligning with the trigger, if available.
templates = [template, template_aligned]
if hastrigger:
template_trigger = vecmeanat(data[:, 0], baseline, selection, trigger, length)
templates.append(template_trigger)
self.templates = np.stack(templates)
self.template_rel_std = np.sqrt(width[1] ** 2 - width[0] ** 2) / center[1]
self.template_N = np.sum(selection)
# For the moving average.
self._cumsum_templates = np.pad(np.cumsum(self.templates, axis=1), [(0,0),(1,0)])
# Compute the noise standard deviation.
STDs = np.std(baseline_zone, axis=1)
self.noise_std = np.mean(STDs[mask])
# Compute the baseline distribution.
bs = baseline[mask]
self.baseline = np.mean(bs)
self.baseline_std = np.std(bs)
# Save filter-aligned template start distribution.
trigarray = start[selection]
self.start_min = np.min(trigarray)
self.start_max = np.max(trigarray)
self.start_median = np.median(trigarray)
return self