def acquire_pulses(filelist,
template,
noisepsd,
tracelength,
thresh,
nchan=2,
trigtemplate=None,
trigthresh=None,
positivepulses=True,
iotype="stanford",
savepath=None,
savename=None,
dumpnum=1,
maxevts=1000,
lgcoverlap=True,
convtoamps=1 / 1024):
"""
Function for running the continuous trigger on many different files and saving the events
to .npz files for later processing.
Parameters
----------
filelist : list of strings
List of files to be opened to take random sections from (should be full paths)
template : ndarray
The pulse template to be used when creating the optimum filter (assumed to be normalized)
noisepsd : ndarray
The two-sided power spectral density in units of A^2/Hz
tracelength : int
The desired trace length (in bins) to be saved when triggering on events.
thresh : float
The number of standard deviations of the energy resolution to use as the threshold for which events
will be detected as a pulse.
trigtemplate : NoneType, ndarray, optional
The template for the trigger channel pulse. If left as None, then the trigger channel will not
be analyzed.
trigthresh : NoneType, float, optional
The threshold value (in units of the trigger channel) such that any amplitudes higher than this will be
detected as ttl trigger event. If left as None, then only the pulses are analyzed.
positivepulses : boolean, optional
Boolean flag for which direction the pulses go in the traces. If they go in the positive direction,
then this should be set to True. If they go in the negative direction, then this should be set to False.
Default is True.
iotype : string, optional
Type of file to open, uses a different IO function. Default is "stanford".
"stanford" : Use qetpy.io.loadstanfordfile to open the files
savepath : NoneType, str, optional
Path to save the events to, if saveevents is True. If this is left as None, then they will
be saved in the current working directory.
savename : NoneType, str, optional
Filename to save the events as. It is recommended that this follows CDMS format, which is
"[code][lasttwodigitsofyear][month][day]_[24hourclocktime]". If this is left as None, then
a dummy filename is used based on the inputted filelist.
dumpnum : int, optional
The dump number that the file should start saving from and the event number should be
determined by when saving. Default is 1.
maxevts : int, optional
The maximum number of events that should be stored in each dump when saving. Default
is 1000.
lgcoverlap : bool, optional
If False, in eventtrigger skip events that overlap, based on tracelength, with the previous event.
convtoamps : float, optional
Correction factor to convert the data to Amps. The traces are multiplied by this
factor, as is the TTL channel (if it exists). Default is 1/1024.
"""
if savepath is None or not savepath:
savepath = "./"
if not savepath.endswith("/"):
savepath += "/"
if savename is None:
now = datetime.datetime.now()
savename = now.strftime("%Y%m%d_%H%M")
if isinstance(filelist, str):
filelist = [filelist]
pulsetimes = np.zeros(maxevts)
pulseamps = np.zeros(maxevts)
trigtimes = np.zeros(maxevts)
trigamps = np.zeros(maxevts)
evttraces = np.zeros((maxevts, nchan, tracelength))
trigtypes = np.zeros((maxevts, 3), dtype=bool)
evt_counter = 0
for f in filelist:
if iotype == "stanford":
traces, times, fs, trig = io.loadstanfordfile(
f, convtoamps=convtoamps)
if trigtemplate is None:
trig = None
else:
raise ValueError("Unrecognized iotype inputted.")
filt = OptimumFilt(fs,
template,
noisepsd,
tracelength,
trigtemplate=trigtemplate,
lgcoverlap=lgcoverlap)
filt.filtertraces(traces, times, trig=trig)
filt.eventtrigger(thresh,
trigthresh=trigthresh,
positivepulses=positivepulses)
numevts = len(filt.pulsetimes)
evt_counter += numevts
if evt_counter < maxevts:
pulsetimes[evt_counter - numevts:evt_counter] = filt.pulsetimes
pulseamps[evt_counter - numevts:evt_counter] = filt.pulseamps
trigtimes[evt_counter - numevts:evt_counter] = filt.trigtimes
trigamps[evt_counter - numevts:evt_counter] = filt.trigamps
evttraces[evt_counter - numevts:evt_counter] = filt.evttraces
trigtypes[evt_counter - numevts:evt_counter] = filt.trigtypes
elif evt_counter >= maxevts:
numextra = evt_counter - maxevts
numtoadd = maxevts - (evt_counter - numevts)
pulsetimes[evt_counter - numevts:] = filt.pulsetimes[:numtoadd]
pulseamps[evt_counter - numevts:] = filt.pulseamps[:numtoadd]
trigtimes[evt_counter - numevts:] = filt.trigtimes[:numtoadd]
trigamps[evt_counter - numevts:] = filt.trigamps[:numtoadd]
evttraces[evt_counter - numevts:] = filt.evttraces[:numtoadd]
trigtypes[evt_counter - numevts:] = filt.trigtypes[:numtoadd]
for ii in range(numextra // maxevts + 1):
io.saveevents_npz(pulsetimes=pulsetimes,
pulseamps=pulseamps,
trigtimes=trigtimes,
trigamps=trigamps,
traces=evttraces,
trigtypes=trigtypes,
savepath=savepath,
savename=savename,
dumpnum=dumpnum)
dumpnum += 1
pulsetimes.fill(0)
pulseamps.fill(0)
trigtimes.fill(0)
trigamps.fill(0)
evttraces.fill(0)
trigtypes.fill(0)
numleft = numevts - (numtoadd + ii * maxevts)
if numleft > maxevts:
numleft = maxevts
if numleft > 0:
pulsetimes[:numleft] = filt.pulsetimes[numtoadd + ii *
maxevts:numtoadd +
ii * maxevts +
numleft]
pulseamps[:numleft] = filt.pulseamps[numtoadd + ii *
maxevts:numtoadd +
ii * maxevts +
numleft]
trigtimes[:numleft] = filt.trigtimes[numtoadd + ii *
maxevts:numtoadd +
ii * maxevts +
numleft]
trigamps[:numleft] = filt.trigamps[numtoadd +
ii * maxevts:numtoadd +
ii * maxevts + numleft]
evttraces[:numleft] = filt.evttraces[numtoadd + ii *
maxevts:numtoadd +
ii * maxevts +
numleft]
trigtypes[:numleft] = filt.trigtypes[numtoadd + ii *
maxevts:numtoadd +
ii * maxevts +
numleft]
evt_counter = np.sum((pulsetimes != 0) | (trigtimes != 0))
# clean up the rest of the events
if evt_counter > 0:
io.saveevents_npz(pulsetimes=pulsetimes,
pulseamps=pulseamps,
trigtimes=trigtimes,
trigamps=trigamps,
traces=evttraces,
trigtypes=trigtypes,
savepath=savepath,
savename=savename,
dumpnum=dumpnum)
def _buildfakepulses_seg(rq,
cut,
templates,
amplitudes,
tdelay,
basepath,
taurises=None,
taufalls=None,
channels="PDS1",
relcal=None,
det="Z1",
convtoamps=1,
fs=625e3,
dumpnum=1,
filetype="mid.gz",
lgcsavefile=False,
savefilepath=None):
"""
Hidden helper function for building fake pulses.
Parameters
----------
rq : pandas.DataFrame
A pandas DataFrame object that contains all of the RQs for the dataset specified.
cut : array_like
A boolean array for the cut that selects the traces that will be loaded from the dump
files. These traces serve as the underlying data to which a template is added.
templates : ndarray, list of ndarray
The template(s) to be added to the traces, assumed to be normalized to a max height
of 1. The template start time should be centered on the center bin. If a list of templates,
then each template will be added to the traces in succession, using the corresponding
`amplitudes` and `tdelay`.
amplitudes : ndarray, list of ndarray
The amplitudes, in Amps, by which to scale the template to add to the traces. Must be
same length as cut. A list of ndarray can be passed, where each ndarray corresponds to
the amplitudes of the corresponding template in the list of templates.
tdelay : ndarray, list of ndarray
The time delay offset, in seconds, by which to shift the template to add to the traces.
Bin interpolation is implemented for values that are not a multiple the reciprocal of
the digitization rate. A list of ndarray can be passed, where each ndarray corresponds to
the tdelays of the corresponding template in the list of templates.
basepath : str
The base path to the directory that contains the folders that the event dumps
are in. The folders in this directory should be the series numbers.
taurises : ndarray, list of ndarray, NoneType
If specified, the rise times for the simulated pulses, in seconds. If left as None, then
the `templates` argument is used.
taufalls : ndarray, list of ndarray, NoneType
If specified, the rise times for the simulated pulses, in seconds. If left as None, then
the `templates` argument is used.
channels : str, list of str, optional
A list of strings that contains all of the channels that should be loaded. Only used if
filetype=='mid.gz'.
det : str, list of str, optional
String or list of strings that specifies the detector name. Only used if
filetype=='mid.gz'. If a list of strings, then should each value should directly
correspond to the channel names. If a string is inputted and there are multiple
channels, then it is assumed that the detector name is the same for each channel.
relcal : ndarray, optional
An array with the amplitude scalings between channels used when making the total
If channels is supplied, relcal indices correspond to that list
Default is all 1 (no relative scaling).
convtoamps : float, optional
The factor that the traces should be multiplied by to convert ADC bins to Amperes.
fs : float, optional
The sample rate in Hz of the data.
filetype : str, optional
The string that corresponds to the file type that will be opened. Supports two
types: "mid.gz" and "npz". "mid.gz" is the default.
lgcsavefile : bool, optional
A boolean flag for whether or not to save the fake data to a file.
savefilepath : str, optional
The string that corresponds to the file path that will be saved.
Returns
-------
None
"""
seriesnumber = list(set(rq.seriesnumber[cut]))[0]
ntraces = np.sum(cut)
t, traces, _ = io.getrandevents(
basepath,
rq.eventnumber,
rq.seriesnumber,
cut=cut,
channels=channels,
det=det,
convtoamps=convtoamps,
fs=fs,
ntraces=ntraces,
filetype=filetype,
)
nchan = traces.shape[1]
nbins = traces.shape[-1]
t = np.arange(nbins) / fs
if relcal is None:
relcal = np.ones(nchan)
elif nchan != len(relcal):
raise ValueError('relcal must have length equal to number of channels')
tracessum = np.sum(traces, axis=1)
fakepulses = np.zeros(traces.shape)
newtrace = np.zeros(tracessum.shape[-1])
for ii in range(ntraces):
newtrace[:] = tracessum[ii]
if taurises is not None and taufalls is not None:
for tr, tf, amp, td in zip(taurises, taufalls, amplitudes, tdelay):
newtrace += amp[ii] * rp.make_ideal_template(
t, tr[ii], tf[ii], offset=td[ii] * fs)
else:
for temp, amp, td in zip(templates, amplitudes, tdelay):
newtrace += amp[ii] * rp.shift(temp, td[ii] * fs)
# multiply by reciprocal of the relative calibration such that when the processing script
# creates the total channel pulse, it will be equal to newtrace
for jj in range(nchan):
if relcal[jj] != 0:
fakepulses[ii, jj] = newtrace / (relcal[jj] * nchan)
if lgcsavefile:
if filetype == 'npz':
savefilename = f"{seriesnumber:010}"
savefilename = savefilename[:6] + '_' + savefilename[6:]
savefilename = savefilename + "_fake_pulses"
truthamps = np.stack(amplitudes, axis=1)
truthtdelay = np.stack(tdelay, axis=1)
trigtypes = np.zeros((ntraces, 3), dtype=bool)
io.saveevents_npz(
traces=fakepulses,
trigtypes=trigtypes,
truthamps=truthamps,
truthtdelay=truthtdelay,
savepath=savefilepath,
savename=savefilename,
dumpnum=dumpnum,
)
elif filetype == "mid.gz":
savefilename = f"{seriesnumber:012}"
savefilename = savefilename[:8] + '_' + savefilename[8:]
if np.issubdtype(type(seriesnumber), np.integer):
snum_str = f"{seriesnumber:012}"
snum_str = snum_str[:8] + '_' + snum_str[8:]
else:
snum_str = seriesnumber
full_settings_dict = getDetectorSettings(f"{basepath}{snum_str}",
"")
settings_dict = {d: full_settings_dict[d] for d in det}
for ch, d in zip(channels, det):
settings_dict[d]["detectorType"] = 710
settings_dict[d]["phononTraceLength"] = int(
settings_dict[d][ch]["binsPerTrace"])
settings_dict[d]["phononPreTriggerLength"] = settings_dict[d][
"phononTraceLength"] // 2
settings_dict[d]["phononSampleRate"] = int(
1 / settings_dict[d][ch]["timePerBin"])
events_list = _create_events_list(
tdelay[0],
amplitudes[0],
fakepulses,
channels,
det,
convtoamps,
seriesnumber,
dumpnum,
)
io.saveevents_midgz(
events=events_list,
settings=settings_dict,
savepath=savefilepath,
savename=savefilename,
dumpnum=dumpnum,
)
else:
raise ValueError('Inputted filetype is not supported.')
def acquire_randoms(filelist,
n,
l,
datashape=None,
iotype="stanford",
savepath=None,
savename=None,
dumpnum=1,
maxevts=1000,
convtoamps=1 / 1024):
"""
Function for acquiring random traces from a list of files and saving the results
to a .npz file for later processing.
Parameters
----------
filelist : list of strings
List of files to be opened to take random sections from (should be full paths)
n : int
Number of sections to choose
l : int
Length in bins of sections
datashape : tuple, NoneType, optional
The shape of the data in each file. If inputted, this should be a tuple that is
(# of traces in a dataset, # of bins in each trace). If left as None, then the first file
in filelist is opened, and the shape of the data in it is used.
iotype : string, optional
Type of file to open, uses a different IO function. Default is "stanford".
"stanford" : Use qetpy.io.loadstanfordfile to open the files
savepath : NoneType, str, optional
Path to save the events to, if saveevents is True. If this is left as None, then they will
be saved in the current working directory.
savename : NoneType, str, optional
Filename to save the events as. It is recommended that this follows CDMS format, which is
"[code][lasttwodigitsofyear][month][day]_[24hourclocktime]". If this is left as None, then
a dummy filename is used based on the inputted filelist.
dumpnum : int, optional
The dump number that the file should start saving from and the event number should be
determined by when saving. Default is 1.
maxevts : int, optional
The maximum number of events that should be stored in each dump when saving. Default
is 1000.
convtoamps : float, optional
Correction factor to convert the data to Amps. The traces are multiplied by this
factor, as is the TTL channel (if it exists). Default is 1/1024.
"""
if savepath is None or not savepath:
savepath = "./"
if not savepath.endswith("/"):
savepath += "/"
if savename is None:
now = datetime.datetime.now()
savename = now.strftime("%Y%m%d_%H%M")
if isinstance(filelist, str):
filelist = [filelist]
if datashape is None:
# get the shape of data from the first dataset, we assume the shape is the same for all files
if iotype == "stanford":
traces = io.loadstanfordfile(filelist[0])[0]
datashape = (traces.shape[0], traces.shape[-1])
else:
raise ValueError("Unrecognized iotype inputted.")
nmax = int(datashape[-1] / l)
choicelist = list(range(len(filelist))) * nmax * datashape[0]
np.random.shuffle(choicelist)
rows = np.array(choicelist[:n])
counts = Counter(rows)
evttimes_list = []
res_list = []
evt_counter = 0
for key in counts.keys():
if iotype == "stanford":
traces, t, fs, _ = io.loadstanfordfile(filelist[key],
convtoamps=convtoamps)
else:
raise ValueError("Unrecognized iotype inputted.")
et, r = rand_sections(traces, counts[key], l, t=t, fs=fs)
evt_counter += len(et)
evttimes_list.append(et)
res_list.append(r)
if evt_counter >= maxevts:
evttimes = np.concatenate(evttimes_list)
res = np.vstack(res_list)
del evttimes_list
del res_list
trigtypes = np.zeros((len(evttimes), 3), dtype=bool)
trigtypes[:, 0] = True
for ii in range(len(evttimes) // maxevts):
io.saveevents_npz(randomstimes=evttimes[:maxevts],
traces=res[:maxevts],
trigtypes=trigtypes[:maxevts],
savepath=savepath,
savename=savename,
dumpnum=dumpnum)
dumpnum += 1
evttimes = evttimes[maxevts:]
res = res[maxevts:]
trigtypes = trigtypes[maxevts:]
if len(evttimes) / maxevts > 1:
evttimes_list = [evttimes]
res_list = [res]
evt_counter = len(evttimes)
else:
evttimes_list = []
res_list = []
evt_counter = 0
# clean up the remaining events
if evt_counter > 0:
evttimes = np.concatenate(evttimes_list)
res = np.vstack(res_list)
del evttimes_list
del res_list
trigtypes = np.zeros((len(evttimes), 3), dtype=bool)
trigtypes[:, 0] = True
for ii in range(np.ceil(len(evttimes) / maxevts).astype(int)):
io.saveevents_npz(randomstimes=evttimes[:maxevts],
traces=res[:maxevts],
trigtypes=trigtypes[:maxevts],
savepath=savepath,
savename=savename,
dumpnum=dumpnum)
dumpnum += 1
if ii + 1 != np.ceil(len(evttimes) / maxevts).astype(int):
evttimes = evttimes[maxevts:]
res = res[maxevts:]
trigtypes = trigtypes[maxevts:]