filter_order = 6
plot_filter = True
power_sum_cut_location = 50 #index
power_sum_cut_value = 13000 #Events with larger power sum then this are ignored.
peak_cut = 60 #At least one channel has to have a signal cross this thresh.
for run_index, run in enumerate(runs):
eventids = known_pulser_ids['run%i' % run]
reader = Reader(datapath, run)
waveform_times = reader.t()
waveforms_upsampled = {}
waveforms_raw = {}
#Prepare filter
reader.setEntry(98958)
wf = reader.wf(0)
wf, waveform_times = scipy.signal.resample(wf,
len(wf) * resample_factor,
t=reader.t())
dt = waveform_times[1] - waveform_times[0]
filter_y, freqs = makeFilter(waveform_times,
crit_freq_low_pass_MHz,
crit_freq_high_pass_MHz,
filter_order,
plot_filter=plot_filter)
try:
for channel in range(8):
waveforms_upsampled['ch%i' % channel] = numpy.zeros(
(len(eventids),
reader.header().buffer_length * resample_factor))
for channel in range(8):
max_corrs['ch%i'%channel] = numpy.zeros(len(eventids))
delays['ch%i'%channel] = numpy.zeros(len(eventids))
'''
max_corrs = numpy.zeros((len(eventids),8))
delays = numpy.zeros((len(eventids),8))
for event_index, eventid in enumerate(eventids):
sys.stdout.write('\r(%i/%i)'%(event_index+1,len(eventids)))
sys.stdout.flush()
reader.setEntry(eventid)
wfs = numpy.zeros((8,original_wf_len))
for channel in range(8):
wfs[channel] = reader.wf(channel)
wfs = scipy.signal.resample(wfs,upsample_wf_len,axis=1)
#Apply filter
wfs = numpy.fft.irfft(numpy.multiply(filter_y,numpy.fft.rfft(wfs,axis=1)),axis=1)
#Can't find a way to vectorize corr
scaled_wfs = wfs/numpy.tile(numpy.std(wfs,axis=1),(numpy.shape(wfs)[1],1)).T
corr = numpy.zeros((8,numpy.shape(wfs)[1]*2 - 1))
for channel in range(8):
corr[channel] = scipy.signal.correlate(templates_scaled_2d[channel],scaled_wfs[channel])/(len_template) #should be roughly normalized between -1,1
max_corrs[event_index] = numpy.max(corr,axis=1)
delays[event_index] = corr_delay_times[numpy.argmax(corr,axis=1)]
calculate_correlation_values = False #If True then the values we be newly calculated, if false then will try to load them from the existing files
#Parameters:
#Curve choice is a parameter in the bi-delta template model that changes the timing of the input dela signal.
curve_choice = 0
upsample_factor = 4
save_data = True
if farm_mode == False:
plt.close('all')
run = int(sys.argv[1])
reader = Reader(datapath,run)
#Prepare for Correlations
reader.setEntry(0)
waveform_times = reader.t()
waveform_sample = reader.wf(0)
waveform_sample, waveform_times = scipy.signal.resample(waveform_sample,len(waveform_sample)*upsample_factor,t=waveform_times) #upsample times to desired amount.
cr_gen = crt.CosmicRayGenerator(waveform_times,t_offset=800.0,model='bi-delta')
template_t, template_E = cr_gen.eFieldGenerator(plot=True,curve_choice=curve_choice)
len_t = len(template_t)
template_E = template_E/(numpy.std(template_E)*len_t) #Pre dividing to handle normalization of cross correlation.
if calculate_correlation_values == True:
try:
print(reader.status())
except Exception as e:
print('Status Tree not present. Returning Error.')
print('\nError in %s'%inspect.stack()[0][3])
hpol_snrs = numpy.zeros(len(eventids))
vpol_snrs = numpy.zeros(len(eventids))
total_snrs = numpy.zeros(len(eventids))
snrs = numpy.zeros(8)
good_antennas = numpy.array([0, 2, 3, 4, 5, 6, 7])
for event_index, eventid in enumerate(eventids):
sys.stdout.write('\r%i/%i' %
(event_index + 1, len(eventids)))
sys.stdout.flush()
reader.setEntry(eventid)
for i in range(8):
wf = reader.wf(i)
snrs[i] = p2p[event_index, i] / numpy.std(wf)
hpol_snrs[event_index] = numpy.max(snrs[0::2])
vpol_snrs[event_index] = numpy.max(
snrs[numpy.array([3, 5, 7])])
total_snrs[event_index] = numpy.max(
snrs[good_antennas])
plt.figure()
plt.hist(hpol_snrs, bins=100)
plt.ylabel('Counts')
plt.xlabel('Max vSNR Across Hpol Channels per Signal')
res = scipy.stats.cumfreq(hpol_snrs, numbins=100)
x = res.lowerlimit + numpy.linspace(
print('%s group already exists in file %s' %
(template_filename_root, filename))
template = numpy.loadtxt(template_filename,
delimiter=',')
template_std = numpy.std(template, axis=1) * len(
reader.t()) #Normalization factor
for eventid in eventids:
if eventid % 1000 == 0:
sys.stdout.write('\r%i/%i' %
(eventid, len(eventids) - 1))
sys.stdout.flush()
reader.setEntry(eventid)
for antenna in range(8):
signal = reader.wf(antenna)
std = numpy.std(signal)
c = scipy.signal.correlate(
template[antenna],
signal) / (std * template_std[antenna])
correlation_values[eventid,
antenna] = numpy.max(c)
if plot == True:
plt.figure()
plt.hist(numpy.max(correlation_values, axis=1),
bins=50)
file['template_correlations'][template_filename_root][
...] = correlation_values
file.close()
def getSpectData(datapath,
run,
event_limit,
bin_size=10,
trigger_type=1,
group_fft=False):
'''
This function obtains the data for a spectrogram.
Parameters
----------
datapath : str
The path to the data where the runs are stored. This is the same as the input to
the reader class.
run : int
The run number to be loaded.
event_limit : int
This limits the number of events to load. Loads from beginning of run to end, so
reducing this speeds up the calculation by cutting off the later portions of the
run.
bin_size : int
This is the number of seconds to include in each time slice of the spectrogram. The
average spectra will be computed per bin. Default is 10.
trigger_type : int
This is the trigger type of events included in the spectrogram. The default is 1.
group_fft : bool
This enables the fft calculation to be performed simultaneously for all events, rather
than per waveform as they are loaded in. This may be faster but requires more memory.
Default is False.
Returns
-------
reader : examples.beacon_data_reader.Reader
This is the reader for the selected run.
freqs : numpy.ndarray of floats
This is the list of frequencies for corresponding to the y-axis of the spectrogram data.
spectra_dbish_binned : dict
This is the data corresponding to the spectrogram. Each entry in the dictionary contains
the spectrogram data for a particular channel. This are returned in dB-like units. I.e.
they are calculated as if the waveforms were in volts, but in reality the waveforms are in
adu. Some there is some offset from these values to true dB units.
'''
reader = Reader(datapath, run)
N = reader.N() if event_limit == None else min(reader.N(),
abs(event_limit))
print('\nReader:')
d = tools.interpret.getReaderDict(reader)
pprint(d)
print('\nHeader:')
h = tools.interpret.getHeaderDict(reader)
pprint(h)
print('\nStatus:')
s = tools.interpret.getStatusDict(reader)
pprint(s)
if reader.N() == 0:
print('No events found in the selected run.')
else:
def rfftWrapper(channel, waveform_times, *args, **kwargs):
spec = numpy.fft.rfft(*args, **kwargs)
real_power_multiplier = 2.0 * numpy.ones_like(
spec
) #The factor of 2 because rfft lost half of the power except for dc and Nyquist bins (handled below).
if len(numpy.shape(spec)) != 1:
real_power_multiplier[:, [0, -1]] = 1.0
else:
real_power_multiplier[[0, -1]] = 1.0
spec_dbish = 10.0 * numpy.log10(
real_power_multiplier * spec * numpy.conj(spec) /
len(waveform_times)
) #10 because doing power in log. Dividing by N to match monutau.
return channel, spec_dbish
waveform_times = reader.t()
freq_step = 1.0 / (len(waveform_times) *
(numpy.diff(waveform_times)[0] * 1e-9))
freqs = numpy.arange(len(waveform_times) // 2 + 1) * freq_step
freq_nyquist = 1 / (2.0 * numpy.diff(waveform_times)[0] * 1e-9)
if group_fft == True:
waveforms = {}
spectra_dbish = {}
readout_times = []
for channel in range(8):
if group_fft == True:
waveforms['ch%i' % channel] = numpy.zeros(
(N, reader.header().buffer_length), dtype=int)
spectra_dbish['ch%i' % channel] = numpy.zeros(
(N, reader.header().buffer_length // 2 + 1), dtype=float)
print('')
for event_index, eventid in enumerate(
range(N if event_limit == None else event_limit)):
sys.stdout.write('\r(%i/%i)' % (eventid + 1, N))
sys.stdout.flush()
reader.setEntry(eventid)
readout_times.append(getattr(reader.header(), 'readout_time'))
for channel in range(8):
if group_fft == True:
waveforms['ch%i' %
channel][event_index] = reader.wf(channel)
else:
spectra_dbish['ch%i' % channel][event_index] = rfftWrapper(
'ch%i' % channel, waveform_times,
reader.wf(channel))[1]
if group_fft == True:
with concurrent.futures.ThreadPoolExecutor(
max_workers=cpu_count()) as executor:
thread_results = []
for channel in range(8):
thread_results.append(
executor.submit(rfftWrapper, 'ch%i' % channel,
waveform_times,
waveforms['ch%i' % channel]))
print('Weaving threads')
sys.stdout.flush()
for index, future in enumerate(
concurrent.futures.as_completed(thread_results)):
spectra_dbish[future.result()[0]] = future.result()[1]
print('%i/8 Channel FFTs Completed' % (index + 1))
bin_edges = numpy.arange(min(readout_times),
max(readout_times) + bin_size, bin_size)
bin_L_2d = numpy.tile(bin_edges[:-1], (len(readout_times), 1))
bin_R_2d = numpy.tile(
numpy.roll(bin_edges, -1)[:-1], (len(readout_times), 1))
readout_times_2d = numpy.tile(readout_times, (len(bin_edges) - 1, 1)).T
cut_2d = numpy.logical_and(readout_times_2d >= bin_L_2d,
readout_times_2d < bin_R_2d).T
del bin_L_2d
del bin_R_2d
del readout_times_2d
spectra_dbish_binned = {}
for channel in range(8):
spectra_dbish_binned['ch%i' % channel] = numpy.zeros(
(len(freqs), len(bin_edges) - 1))
for index, cut in enumerate(cut_2d):
spectra_dbish_binned['ch%i' % channel][:, index] = numpy.mean(
spectra_dbish['ch%i' % channel][cut], axis=0)
spectra_dbish_binned['ch%i' % channel] = numpy.flipud(
numpy.ma.array(spectra_dbish_binned['ch%i' % channel],
mask=numpy.isnan(
spectra_dbish_binned['ch%i' % channel])))
return reader, freqs, spectra_dbish_binned
calculate_correlation_values = False #If True then the values we be newly calculated, if false then will try to load them from the existing files
#Parameters:
#Curve choice is a parameter in the bi-delta template model that changes the timing of the input dela signal.
curve_choice = 0
upsample_factor = 4
save_data = True
if farm_mode == False:
plt.close('all')
run = int(sys.argv[1])
reader = Reader(datapath, run)
#Prepare for Correlations
reader.setEntry(0)
waveform_times = reader.t()
waveform_sample = reader.wf(0)
waveform_sample, waveform_times = scipy.signal.resample(
waveform_sample,
len(waveform_sample) * upsample_factor,
t=waveform_times) #upsample times to desired amount.
cr_gen = crt.CosmicRayGenerator(waveform_times,
t_offset=800.0,
model='bi-delta')
template_t, template_E = cr_gen.eFieldGenerator(
plot=True, curve_choice=curve_choice)
len_t = len(template_t)
template_E = template_E / (
numpy.std(template_E) * len_t
) #Pre dividing to handle normalization of cross correlation.
meas = {}
for channel in range(8):
meas[channel] = []
for event_index, eventid in enumerate(eventids):
if eventid < event_min:
continue
sys.stdout.write('(%i/%i)\r' % (event_index, len(eventids)))
sys.stdout.flush()
reader.setEntry(eventid)
event_times = reader.t()
for channel in range(8):
channel = int(channel)
meas[channel].append(
numpy.max(reader.wf(channel)) -
numpy.min(reader.wf(channel)))
if run == 1507:
lines_of_interest = numpy.array(
[1035, 1275, 1755, 10034, 11714, 12314, 12914, 15711, 17354])
elif run == 1509:
lines_of_interest = numpy.array(
[721, 1201, 1801, 2401, 3722, 4082, 4201])
elif run == 1511:
lines_of_interest = numpy.array([892, 2690, 3892, 5812])
plt.figure()
plt.title(run)
for channel in range(8):
pprint(tools.interpret.getReaderDict(reader))
print('\nHeader:')
pprint(tools.interpret.getHeaderDict(reader))
print('\nStatus:')
pprint(tools.interpret.getStatusDict(reader))
reader.header().Dump()
reader.status().Dump()
#print reader.N()
# plot all waveforms
plt.figure()
for i in range(4):
if i == 0:
ax = plt.subplot(4, 1, i + 1)
plt.plot(reader.t() - 5200, reader.wf(2 * i))
else:
plt.subplot(4, 1, i + 1, sharex=ax, sharey=ax)
plt.plot(reader.t() - 5200, reader.wf(2 * i))
if i in [0, 1, 2, 3]:
plt.ylabel('V (adu)')
if i == 3:
plt.xlabel('t (ns)')
plt.suptitle('Run %i, Event %i, Hpol' % (run, eid))
plt.xlim(0, 600)
plt.minorticks_on()
plt.grid(b=True, which='major', color='k', linestyle='-')
plt.grid(b=True,
which='minor',
color='tab:gray',
linestyle='--',
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
if __name__ == '__main__':
try:
plt.close('all')
#Get timing info from real BEACON data for testing.
run = 1509
known_pulser_ids = info.loadPulserEventids(remove_ignored=True)
eventid = known_pulser_ids['run%i' % run]['hpol'][0]
reader = Reader(datapath, run)
reader.setEntry(eventid)
test_t = reader.t()
test_pulser_adu = reader.wf(0)
#Creating test signal
cr_gen = CosmicRayGenerator(test_t, t_offset=800.0, model='bi-delta')
for curve_choice in range(4):
out_t, out_E = cr_gen.eFieldGenerator(plot=True,
curve_choice=curve_choice)
plt.figure()
plt.subplot(2, 1, 1)
plt.plot(test_t, test_pulser_adu, label='Pulser Signal')
plt.ylabel('E (adu)')
plt.xlabel('t (ns)')
plt.legend()
plt.minorticks_on()
'impulsivity_h')
roi_impulsivity_sort = numpy.argsort(
roi_impulsivity
)[::-1] #Reverse sorting so high numbers are first.
if len(roi_eventids) > limit_events:
print(
'LIMITING TIME DELAY CALCULATION TO %i MOST IMPULSIVE EVENTS'
% limit_events)
roi_eventids = numpy.sort(
roi_eventids[roi_impulsivity_sort[0:limit_events]])
argmaxs = []
reader.setEntry(roi_eventids[0])
for channel in range(8):
argmaxs.append(numpy.argmax(reader.wf(channel)))
waveform_index_range = (int(numpy.min(argmaxs) - 40),
int(numpy.max(argmaxs) + 100))
tdc = TimeDelayCalculator(
reader,
final_corr_length=final_corr_length,
crit_freq_low_pass_MHz=None,
crit_freq_high_pass_MHz=None,
low_pass_filter_order=None,
high_pass_filter_order=None,
waveform_index_range=waveform_index_range,
plot_filters=False,
apply_phase_response=apply_phase_response)
if sine_subtract:
trig_eventids = eventids[
trigger_type == trig_type] #All eventids of this trig type
trig_eventids = numpy.sort(
numpy.random.choice(trig_eventids, 2)
) #Randomly choosing a subset and sorting for faster loading of events
for eventid in trig_eventids:
reader.setEntry(
eventid
) #Actually makes the wf function adress the correct event.
fig = plt.figure()
fig.canvas.set_window_title('Run %i event %i, Trig Type %i' %
(run, eventid, trig_type))
plt.title('Run %i event %i, Trig Type %i' %
(run, eventid, trig_type))
plt.subplot(2, 1, 1)
plt.ylabel('Hpol Adu')
plt.subplot(2, 1, 2)
plt.ylabel('Vpol Adu')
plt.xlabel('t (ns)')
for channel in range(8):
plt.subplot(2, 1, channel % 2 + 1)
wf = reader.wf(channel)
plt.plot(times, wf, label='Ch %i' % channel)
plt.subplot(2, 1, 1)
plt.legend(loc='upper left')
plt.subplot(2, 1, 2)
plt.legend(loc='upper left')
exclude_eventids = []
waveforms_corr = {}
upsampled_waveforms = {}
for channel in channels:
channel=int(channel)
waveforms_corr['ch%i'%channel] = numpy.zeros((len(eventids),len(waveform_times_corr)))
upsampled_waveforms['ch%i'%channel] = numpy.zeros((len(eventids),final_corr_length//2))
for event_index, eventid in enumerate(eventids):
sys.stdout.write('(%i/%i)\r'%(event_index,len(eventids)))
sys.stdout.flush()
reader.setEntry(eventid)
event_times = reader.t()
for channel in channels:
channel=int(channel)
waveforms_corr['ch%i'%channel][event_index][0:reader.header().buffer_length] = reader.wf(channel)
#Below are the actual time domain waveforms_corr and should not have the factor of 2 padding. The small rounding padding sticks around, so using waveform_times_padded_to_power2 times,
if use_filter:
upsampled_waveforms['ch%i'%channel][event_index] = numpy.fft.irfft(numpy.multiply(filter_y_wf,numpy.fft.rfft(waveforms_corr['ch%i'%channel][event_index][0:len(waveform_times_padded_to_power2)])),n=final_corr_length//2) * ((final_corr_length//2)/len(waveform_times_padded_to_power2))
else:
upsampled_waveforms['ch%i'%channel][event_index] = numpy.fft.irfft(numpy.fft.rfft(waveforms_corr['ch%i'%channel][event_index][0:len(waveform_times_padded_to_power2)]),n=final_corr_length//2) * ((final_corr_length//2)/len(waveform_times_padded_to_power2))
#upsampled_waveforms['ch%i'%channel][event_index], upsampled_times = scipy.signal.resample(waveforms_corr['ch%i'%channel][event_index][0:len(waveform_times_padded_to_power2)],final_corr_length//2,t=waveform_times_padded_to_power2)
'''
print('Upsampling waveforms_corr')
for channel in channels:
print(channel)
channel=int(channel)
upsampled_waveforms['ch%i'%channel], upsampled_times = scipy.signal.resample(waveforms_corr['ch%i'%channel],2*(final_corr_length//2 + 1),t=waveform_times_corr,axis = 1)
'''
print('\n')