def test_3d_index_histograming(self): # check compiled hist_3D_index function
with tb.open_file(os.path.join(tests_data_folder + '/hist_data.h5'), mode="r") as in_file_h5:
xyz = in_file_h5.root.HistDataXYZ[:]
x, y, z = xyz[0], xyz[1], xyz[2]
shape = (100, 100, 100)
array_fast = analysis_utils.hist_3d_index(x, y, z, shape=shape)
array = np.histogramdd(np.column_stack((x, y, z)), bins=shape, range=[[0, shape[0] - 1], [0, shape[1] - 1], [0, shape[2] - 1]])[0]
shape = (50, 200, 200) # shape that is too small for the indices to trigger exception
exception_ok = False
try:
array_fast = analysis_utils.hist_3d_index(x, y, z, shape=shape)
except IndexError:
exception_ok = True
except: # other exception that should not occur
pass
self.assertTrue(exception_ok & np.all(array == array_fast))
def analyze_hit_delay(raw_data_file):
# Interpret data and create hit table
with AnalyzeRawData(raw_data_file=raw_data_file,
create_pdf=False) as analyze_raw_data:
analyze_raw_data.create_occupancy_hist = False # Too many scan parameters to do in ram histogramming
analyze_raw_data.create_hit_table = True
analyze_raw_data.interpreter.set_warning_output(
False) # A lot of data produces unknown words
analyze_raw_data.interpret_word_table()
analyze_raw_data.interpreter.print_summary()
# Store calibration values in variables
vcal_c0 = analyze_raw_data.vcal_c0
vcal_c1 = analyze_raw_data.vcal_c1
c_high = analyze_raw_data.c_high
# Create relative BCID and mean relative BCID histogram for each pixel / injection delay / PlsrDAC setting
with tb.open_file(raw_data_file + '_analyzed.h5', mode="w") as out_file_h5:
hists_folder = out_file_h5.create_group(out_file_h5.root,
'PixelHistsMeanRelBcid')
hists_folder_2 = out_file_h5.create_group(out_file_h5.root,
'PixelHistsRelBcid')
hists_folder_3 = out_file_h5.create_group(out_file_h5.root,
'PixelHistsTot')
hists_folder_4 = out_file_h5.create_group(out_file_h5.root,
'PixelHistsMeanTot')
hists_folder_5 = out_file_h5.create_group(out_file_h5.root, 'HistsTot')
def store_bcid_histograms(bcid_array, tot_array, tot_pixel_array):
logging.debug('Store histograms for PlsrDAC ' + str(old_plsr_dac))
bcid_mean_array = np.average(
bcid_array, axis=3, weights=range(0, 16)
) * sum(range(0, 16)) / np.sum(bcid_array, axis=3).astype(
'f4') # calculate the mean BCID per pixel and scan parameter
tot_pixel_mean_array = np.average(
tot_pixel_array, axis=3, weights=range(0, 16)
) * sum(range(0, 16)) / np.sum(tot_pixel_array, axis=3).astype(
'f4') # calculate the mean tot per pixel and scan parameter
bcid_mean_result = np.swapaxes(bcid_mean_array, 0, 1)
bcid_result = np.swapaxes(bcid_array, 0, 1)
tot_pixel_result = np.swapaxes(tot_pixel_array, 0, 1)
tot_mean_pixel_result = np.swapaxes(tot_pixel_mean_array, 0, 1)
out = out_file_h5.create_carray(
hists_folder,
name='HistPixelMeanRelBcidPerDelayPlsrDac_%03d' % old_plsr_dac,
title=
'Mean relative BCID hist per pixel and different PlsrDAC delays for PlsrDAC '
+ str(old_plsr_dac),
atom=tb.Atom.from_dtype(bcid_mean_result.dtype),
shape=bcid_mean_result.shape,
filters=tb.Filters(complib='blosc',
complevel=5,
fletcher32=False))
out.attrs.dimensions = 'column, row, injection delay'
out.attrs.injection_delay_values = injection_delay
out[:] = bcid_mean_result
out_2 = out_file_h5.create_carray(
hists_folder_2,
name='HistPixelRelBcidPerDelayPlsrDac_%03d' % old_plsr_dac,
title=
'Relative BCID hist per pixel and different PlsrDAC delays for PlsrDAC '
+ str(old_plsr_dac),
atom=tb.Atom.from_dtype(bcid_result.dtype),
shape=bcid_result.shape,
filters=tb.Filters(complib='blosc',
complevel=5,
fletcher32=False))
out_2.attrs.dimensions = 'column, row, injection delay, relative bcid'
out_2.attrs.injection_delay_values = injection_delay
out_2[:] = bcid_result
out_3 = out_file_h5.create_carray(
hists_folder_3,
name='HistPixelTotPerDelayPlsrDac_%03d' % old_plsr_dac,
title=
'Tot hist per pixel and different PlsrDAC delays for PlsrDAC '
+ str(old_plsr_dac),
atom=tb.Atom.from_dtype(tot_pixel_result.dtype),
shape=tot_pixel_result.shape,
filters=tb.Filters(complib='blosc',
complevel=5,
fletcher32=False))
out_3.attrs.dimensions = 'column, row, injection delay'
out_3.attrs.injection_delay_values = injection_delay
out_3[:] = tot_pixel_result
out_4 = out_file_h5.create_carray(
hists_folder_4,
name='HistPixelMeanTotPerDelayPlsrDac_%03d' % old_plsr_dac,
title=
'Mean tot hist per pixel and different PlsrDAC delays for PlsrDAC '
+ str(old_plsr_dac),
atom=tb.Atom.from_dtype(tot_mean_pixel_result.dtype),
shape=tot_mean_pixel_result.shape,
filters=tb.Filters(complib='blosc',
complevel=5,
fletcher32=False))
out_4.attrs.dimensions = 'column, row, injection delay'
out_4.attrs.injection_delay_values = injection_delay
out_4[:] = tot_mean_pixel_result
out_5 = out_file_h5.create_carray(
hists_folder_5,
name='HistTotPlsrDac_%03d' % old_plsr_dac,
title='Tot histogram for PlsrDAC ' + str(old_plsr_dac),
atom=tb.Atom.from_dtype(tot_array.dtype),
shape=tot_array.shape,
filters=tb.Filters(complib='blosc',
complevel=5,
fletcher32=False))
out_5.attrs.injection_delay_values = injection_delay
out_5[:] = tot_array
old_plsr_dac = None
# Get scan parameters from interpreted file
with tb.open_file(raw_data_file + '_interpreted.h5',
'r') as in_file_h5:
scan_parameters_dict = get_scan_parameter(
in_file_h5.root.meta_data[:])
plsr_dac = scan_parameters_dict['PlsrDAC']
hists_folder._v_attrs.plsr_dac_values = plsr_dac
hists_folder_2._v_attrs.plsr_dac_values = plsr_dac
hists_folder_3._v_attrs.plsr_dac_values = plsr_dac
hists_folder_4._v_attrs.plsr_dac_values = plsr_dac
injection_delay = scan_parameters_dict[scan_parameters_dict.keys(
)[1]] # injection delay par name is unknown and should be in the inner loop
scan_parameters = scan_parameters_dict.keys()
bcid_array = np.zeros((80, 336, len(injection_delay), 16),
dtype=np.uint16) # bcid array of actual PlsrDAC
tot_pixel_array = np.zeros(
(80, 336, len(injection_delay), 16),
dtype=np.uint16) # tot pixel array of actual PlsrDAC
tot_array = np.zeros((16, ),
dtype=np.uint32) # tot array of actual PlsrDAC
logging.info('Store histograms for PlsrDAC values ' + str(plsr_dac))
progress_bar = progressbar.ProgressBar(widgets=[
'',
progressbar.Percentage(), ' ',
progressbar.Bar(marker='*', left='|', right='|'), ' ',
progressbar.AdaptiveETA()
],
maxval=max(plsr_dac) -
min(plsr_dac),
term_width=80)
for index, (parameters, hits) in enumerate(
get_hits_of_scan_parameter(raw_data_file + '_interpreted.h5',
scan_parameters,
try_speedup=True,
chunk_size=10000000)):
if index == 0:
progress_bar.start(
) # Start after the event index is created to get reasonable ETA
actual_plsr_dac, actual_injection_delay = parameters[
0], parameters[1]
column, row, rel_bcid, tot = hits['column'] - 1, hits[
'row'] - 1, hits['relative_BCID'], hits['tot']
bcid_array_fast = hist_3d_index(column,
row,
rel_bcid,
shape=(80, 336, 16))
tot_pixel_array_fast = hist_3d_index(column,
row,
tot,
shape=(80, 336, 16))
tot_array_fast = hist_1d_index(tot, shape=(16, ))
if old_plsr_dac != actual_plsr_dac: # Store the data of the actual PlsrDAC value
if old_plsr_dac: # Special case for the first PlsrDAC setting
store_bcid_histograms(bcid_array, tot_array,
tot_pixel_array)
progress_bar.update(old_plsr_dac - min(plsr_dac))
# Reset the histrograms for the next PlsrDAC setting
bcid_array = np.zeros((80, 336, len(injection_delay), 16),
dtype=np.uint16)
tot_pixel_array = np.zeros((80, 336, len(injection_delay), 16),
dtype=np.uint16)
tot_array = np.zeros((16, ), dtype=np.uint32)
old_plsr_dac = actual_plsr_dac
injection_delay_index = np.where(
np.array(injection_delay) == actual_injection_delay)[0][0]
bcid_array[:, :, injection_delay_index, :] += bcid_array_fast
tot_pixel_array[:, :,
injection_delay_index, :] += tot_pixel_array_fast
tot_array += tot_array_fast
store_bcid_histograms(
bcid_array, tot_array,
tot_pixel_array) # save histograms of last PlsrDAC setting
progress_bar.finish()
# Take the mean relative BCID histogram of each PlsrDAC value and calculate the delay for each pixel
with tb.open_file(raw_data_file + '_analyzed.h5', mode="r+") as in_file_h5:
hists_folder = in_file_h5.create_group(in_file_h5.root,
'PixelHistsBcidJumps')
plsr_dac_values = in_file_h5.root.PixelHistsMeanRelBcid._v_attrs.plsr_dac_values
# Info output with progressbar
logging.info(
'Detect BCID jumps with pixel based S-Curve fits for PlsrDACs ' +
str(plsr_dac_values))
progress_bar = progressbar.ProgressBar(widgets=[
'',
progressbar.Percentage(), ' ',
progressbar.Bar(marker='*', left='|', right='|'), ' ',
progressbar.AdaptiveETA()
],
maxval=len(plsr_dac_values),
term_width=80)
progress_bar.start()
for index, node in enumerate(
in_file_h5.root.PixelHistsMeanRelBcid
): # loop over all mean relative BCID hists for all PlsrDAC values and determine the BCID jumps
actual_plsr_dac = int(re.search(
r'\d+', node.name).group()) # actual node plsr dac value
# Select the S-curves and interpolate Nans
pixel_data = node[:, :, :]
pixel_data_fixed = pixel_data.reshape(
pixel_data.shape[0] * pixel_data.shape[1] *
pixel_data.shape[2]) # Reshape for interpolation of Nans
nans, x = ~np.isfinite(pixel_data_fixed), lambda z: z.nonzero()[0]
pixel_data_fixed[nans] = np.interp(
x(nans), x(~nans), pixel_data_fixed[~nans]) # interpolate Nans
pixel_data_fixed = pixel_data_fixed.reshape(
pixel_data.shape[0], pixel_data.shape[1],
pixel_data.shape[2]) # Reshape after interpolation of Nans
# Fit all BCID jumps per pixel (1 - 2 jumps expected) with multithreading
pixel_data_shaped = pixel_data_fixed.reshape(
pixel_data_fixed.shape[0] * pixel_data_fixed.shape[1],
pixel_data_fixed.shape[2]).tolist()
pool = mp.Pool(
) # create as many workers as physical cores are available
result_array = np.array(pool.map(fit_bcid_jumps,
pixel_data_shaped))
pool.close()
pool.join()
result_array = result_array.reshape(pixel_data_fixed.shape[0],
pixel_data_fixed.shape[1], 4)
# Store array to file
out = in_file_h5.create_carray(
hists_folder,
name='PixelHistsBcidJumpsPlsrDac_%03d' % actual_plsr_dac,
title='BCID jumps per pixel for PlsrDAC ' +
str(actual_plsr_dac),
atom=tb.Atom.from_dtype(result_array.dtype),
shape=result_array.shape,
filters=tb.Filters(complib='blosc',
complevel=5,
fletcher32=False))
out.attrs.dimensions = 'column, row, BCID first jump, delay first jump, BCID second jump, delay second jump'
out[:] = result_array
progress_bar.update(index)
# Calibrate the step size of the injection delay and create absolute and relative (=time walk) hit delay histograms
with tb.open_file(raw_data_file + '_analyzed.h5',
mode="r+") as out_file_h5:
# Calculate injection delay step size using the average difference of two Scurves of all pixels and plsrDAC settings and the minimum BCID to fix the absolute time scale
differences = np.zeros(
shape=(336, 80,
sum(1 for _ in out_file_h5.root.PixelHistsBcidJumps)),
dtype=np.float)
min_bcid = 15
for index, node in enumerate(
out_file_h5.root.PixelHistsBcidJumps
): # Loop to get last node (the node with most charge injected)
pixel_data = node[:, :, :]
selection = (np.logical_and(pixel_data[:, :, 0] > 0,
pixel_data[:, :, 2] > 0)
) # select pixels with two Scurve fits
difference = np.zeros_like(differences[:, :, 0])
difference[selection] = pixel_data[selection, 3] - pixel_data[
selection,
1] # Difference in delay settings between the scurves
difference[np.logical_or(
difference < 15, difference > 60
)] = 0 # Get rid of bad data leading to difference that is too small / large
differences[:, :, index] = difference
if np.any(pixel_data[selection, 0]) and np.min(
pixel_data[selection, 0]
) < min_bcid: # Search for the minimum rel. BCID delay (= fastes hits)
min_bcid = np.amin(pixel_data[selection, 0])
differences = np.ma.masked_where(
np.logical_or(differences == 0, ~np.isfinite(differences)),
differences)
step_size = np.ma.median(differences) # Delay steps needed for 25 ns
step_size_error = np.ma.std(
differences) # Delay steps needed for 25 ns
logging.info(
'Mean step size for the PLsrDAC delay is %1.2f +- %1.2f ns',
25. / step_size, 25. / step_size**2 * step_size_error)
# Calculate the hit delay per pixel
plsr_dac_values = out_file_h5.root.PixelHistsMeanRelBcid._v_attrs.plsr_dac_values
hit_delay = np.zeros(shape=(336, 80, len(plsr_dac_values)),
dtype=np.float) # Result array
for node in out_file_h5.root.PixelHistsBcidJumps: # loop over all BCID jump hists for all PlsrDAC values to calculate the hit delay
actual_plsr_dac = int(re.search(
r'\d+', node.name).group()) # actual node plsr dac value
plsr_dac_index = np.where(plsr_dac_values == actual_plsr_dac)[0][0]
pixel_data = node[:, :, :]
actual_hit_delay = (pixel_data[:, :, 0] - min_bcid + 1
) * 25. - pixel_data[:, :, 1] * 25. / step_size
hit_delay[:, :, plsr_dac_index] = actual_hit_delay
hit_delay = np.ma.masked_less(hit_delay, 0)
timewalk = hit_delay - np.amin(
hit_delay, axis=2
)[:, :, np.
newaxis] # Time walk calc. by normalization to minimum hit delay for every pixel
# Calculate the mean TOT per PlsrDAC (additional information, not needed for hit delay)
tot = np.zeros(shape=(len(plsr_dac_values), ),
dtype=np.float16) # Result array
for node in out_file_h5.root.HistsTot: # Loop over tot hist for all PlsrDAC values
plsr_dac = int(re.search(r'\d+', node.name).group())
plsr_dac_index = np.where(plsr_dac_values == plsr_dac)[0][0]
tot_data = node[:]
tot[plsr_dac_index] = get_mean_from_histogram(tot_data, range(16))
# Store the data
out = out_file_h5.create_carray(
out_file_h5.root,
name='HistPixelTimewalkPerPlsrDac',
title='Time walk per pixel and PlsrDAC',
atom=tb.Atom.from_dtype(timewalk.dtype),
shape=timewalk.shape,
filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_2 = out_file_h5.create_carray(
out_file_h5.root,
name='HistPixelHitDelayPerPlsrDac',
title='Hit delay per pixel and PlsrDAC',
atom=tb.Atom.from_dtype(hit_delay.dtype),
shape=hit_delay.shape,
filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_3 = out_file_h5.create_carray(out_file_h5.root,
name='HistTotPerPlsrDac',
title='Tot per PlsrDAC',
atom=tb.Atom.from_dtype(tot.dtype),
shape=tot.shape,
filters=tb.Filters(complib='blosc',
complevel=5,
fletcher32=False))
out.attrs.dimensions = 'column, row, PlsrDAC'
out.attrs.delay_calibration = step_size
out.attrs.delay_calibration_error = step_size_error
out.attrs.plsr_dac_values = plsr_dac_values
out_2.attrs.dimensions = 'column, row, PlsrDAC'
out_2.attrs.delay_calibration = step_size
out_2.attrs.delay_calibration_error = step_size_error
out_2.attrs.plsr_dac_values = plsr_dac_values
out_3.attrs.dimensions = 'PlsrDAC'
out_3.attrs.plsr_dac_values = plsr_dac_values
out[:] = timewalk.filled(fill_value=np.NaN)
out_2[:] = hit_delay.filled(fill_value=np.NaN)
out_3[:] = tot
# Mask the pixels that have non valid data and create plots with the time walk and hit delay for all pixels
with tb.open_file(raw_data_file + '_analyzed.h5', mode="r") as in_file_h5:
def plsr_dac_to_charge(
plsr_dac, vcal_c0, vcal_c1,
c_high): # Calibration values are taken from file
voltage = vcal_c0 + vcal_c1 * plsr_dac
return voltage * c_high / 0.16022
def plot_hit_delay(hist_3d,
charge_values,
title,
xlabel,
ylabel,
filename,
threshold=None,
tot_values=None):
# Interpolate tot values for second tot axis
interpolation = interp1d(tot_values,
charge_values,
kind='slinear',
bounds_error=True)
tot = np.arange(16)
tot = tot[np.logical_and(tot >= np.min(tot_values),
tot <= np.max(tot_values))]
array = np.transpose(hist_3d, axes=(2, 1, 0)).reshape(
hist_3d.shape[2], hist_3d.shape[0] * hist_3d.shape[1])
y = np.mean(array, axis=1)
y_err = np.std(array, axis=1)
fig = Figure()
FigureCanvas(fig)
ax = fig.add_subplot(111)
fig.patch.set_facecolor('white')
ax.grid(True)
ax.set_xlabel(xlabel)
ax.set_ylabel(ylabel)
ax.set_xlim((0, np.max(charge_values)))
ax.set_ylim((np.min(y - y_err), np.max(y + y_err)))
ax.plot(charge_values, y, '.-', color='black', label=title)
if threshold is not None:
ax.plot(
[threshold, threshold],
[np.min(y - y_err), np.max(y + y_err)],
linestyle='--',
color='black',
label='Threshold\n%d e' % (threshold))
ax.fill_between(charge_values,
y - y_err,
y + y_err,
color='gray',
alpha=0.5,
facecolor='gray',
label='RMS')
ax2 = ax.twiny()
ax2.set_xlabel("ToT")
ticklab = ax2.xaxis.get_ticklabels()[0]
trans = ticklab.get_transform()
ax2.xaxis.set_label_coords(np.max(charge_values),
1,
transform=trans)
ax2.set_xlim(ax.get_xlim())
ax2.set_xticks(interpolation(tot))
ax2.set_xticklabels([str(int(i)) for i in tot])
ax.text(0.5,
1.07,
title,
horizontalalignment='center',
fontsize=18,
transform=ax2.transAxes)
ax.legend()
filename.savefig(fig)
plsr_dac_values = in_file_h5.root.PixelHistsMeanRelBcid._v_attrs.plsr_dac_values
charge_values = plsr_dac_to_charge(np.array(plsr_dac_values), vcal_c0,
vcal_c1, c_high)
hist_timewalk = in_file_h5.root.HistPixelTimewalkPerPlsrDac[:, :, :]
hist_hit_delay = in_file_h5.root.HistPixelHitDelayPerPlsrDac[:, :, :]
tot = in_file_h5.root.HistTotPerPlsrDac[:]
hist_timewalk = np.ma.masked_invalid(hist_timewalk)
hist_hit_delay = np.ma.masked_invalid(hist_hit_delay)
output_pdf = PdfPages(raw_data_file + '_analyzed.pdf')
plot_hit_delay(np.swapaxes(hist_timewalk, 0, 1),
charge_values=charge_values,
title='Time walk',
xlabel='Charge [e]',
ylabel='Time walk [ns]',
filename=output_pdf,
threshold=np.amin(charge_values),
tot_values=tot)
plot_hit_delay(np.swapaxes(hist_hit_delay, 0, 1),
charge_values=charge_values,
title='Hit delay',
xlabel='Charge [e]',
ylabel='Hit delay [ns]',
filename=output_pdf,
threshold=np.amin(charge_values),
tot_values=tot)
plot_scurves(np.swapaxes(hist_timewalk, 0, 1),
scan_parameters=charge_values,
title='Timewalk of the FE-I4',
scan_parameter_name='Charge [e]',
ylabel='Timewalk [ns]',
min_x=0,
filename=output_pdf)
plot_scurves(
np.swapaxes(hist_hit_delay[:, :, :], 0, 1),
scan_parameters=charge_values,
title='Hit delay (T0) with internal charge injection\nof the FE-I4',
scan_parameter_name='Charge [e]',
ylabel='Hit delay [ns]',
min_x=0,
filename=output_pdf)
for i in [
0, 1,
len(plsr_dac_values) / 4,
len(plsr_dac_values) / 2, -1
]: # Plot 2d hist at min, 1/4, 1/2, max PlsrDAC setting
plot_three_way(hist_timewalk[:, :, i],
title='Time walk at %.0f e' % (charge_values[i]),
x_axis_title='Time walk [ns]',
filename=output_pdf)
plot_three_way(
hist_hit_delay[:, :, i],
title='Hit delay (T0) with internal charge injection at %.0f e'
% (charge_values[i]),
x_axis_title='Hit delay [ns]',
minimum=np.amin(hist_hit_delay[:, :, i]),
maximum=np.max(hist_hit_delay[:, :, i]),
filename=output_pdf)
output_pdf.close()
def histogram_tdc_hits(input_file_hits, hit_selection_conditions, event_status_select_mask, event_status_condition, calibration_file=None, correct_calibration=None, max_tdc=1000, ignore_disabled_regions=True, n_bins=200, plot_data=True):
for condition in hit_selection_conditions:
logging.info('Histogram TDC hits with %s', condition)
def get_charge(max_tdc, tdc_calibration_values, tdc_pixel_calibration): # return the charge from calibration
charge_calibration = np.zeros(shape=(80, 336, max_tdc))
for column in range(80):
for row in range(336):
actual_pixel_calibration = tdc_pixel_calibration[column, row, :]
# Only take pixels with at least 3 valid calibration points
if np.count_nonzero(actual_pixel_calibration != 0) > 2 and np.count_nonzero(np.isfinite(actual_pixel_calibration)) > 2:
selected_measurements = np.isfinite(actual_pixel_calibration) # Select valid calibration steps
selected_actual_pixel_calibration = actual_pixel_calibration[selected_measurements]
selected_tdc_calibration_values = tdc_calibration_values[selected_measurements]
interpolation = interp1d(x=selected_actual_pixel_calibration, y=selected_tdc_calibration_values, kind='slinear', bounds_error=False, fill_value=0)
charge_calibration[column, row, :] = interpolation(np.arange(max_tdc))
return charge_calibration
def plot_tdc_tot_correlation(data, condition, output_pdf):
logging.info('Plot correlation histogram for %s', condition)
plt.clf()
data = np.ma.array(data, mask=(data <= 0))
if np.ma.any(data > 0):
cmap = cm.get_cmap('jet', 200)
cmap.set_bad('w')
plt.title('Correlation with %s' % condition)
norm = colors.LogNorm()
z_max = data.max(fill_value=0)
plt.xlabel('TDC')
plt.ylabel('TOT')
im = plt.imshow(data, cmap=cmap, norm=norm, aspect='auto', interpolation='nearest') # , norm=norm)
divider = make_axes_locatable(plt.gca())
plt.gca().invert_yaxis()
cax = divider.append_axes("right", size="5%", pad=0.1)
plt.colorbar(im, cax=cax, ticks=np.linspace(start=0, stop=z_max, num=9, endpoint=True))
output_pdf.savefig()
else:
logging.warning('No data for correlation plotting for %s', condition)
def plot_hits_per_condition(output_pdf):
logging.info('Plot hits selection efficiency histogram for %d conditions', len(hit_selection_conditions) + 2)
labels = ['All Hits', 'Hits of\ngood events']
for condition in hit_selection_conditions:
condition = re.sub('[&]', '\n', condition)
condition = re.sub('[()]', '', condition)
labels.append(condition)
plt.clf()
plt.bar(range(len(n_hits_per_condition)), n_hits_per_condition, align='center')
plt.xticks(range(len(n_hits_per_condition)), labels, size=8)
plt.title('Number of hits for different cuts')
plt.yscale('log')
plt.ylabel('#')
plt.grid()
for x, y in zip(np.arange(len(n_hits_per_condition)), n_hits_per_condition):
plt.annotate('%d' % (float(y) / float(n_hits_per_condition[0]) * 100.) + r'%', xy=(x, y / 2.), xycoords='data', color='grey', size=15)
output_pdf.savefig()
def plot_corrected_tdc_hist(x, y, title, output_pdf, point_style='-'):
logging.info('Plot TDC hist with TDC calibration')
plt.clf()
y /= np.amax(y) if y.shape[0] > 0 else y
plt.plot(x, y, point_style)
plt.title(title, size=10)
plt.xlabel('Charge [PlsrDAC]')
plt.ylabel('Count [a.u.]')
plt.grid()
output_pdf.savefig()
def get_calibration_correction(tdc_calibration, tdc_calibration_values, filename_new_calibration): # correct the TDC calibration with the TDC calib in filename_new_calibration by shifting the means
with tb.open_file(filename_new_calibration, 'r') as in_file_2:
charge_calibration_1, charge_calibration_2 = tdc_calibration, in_file_2.root.HitOrCalibration[:, :, :, 1]
plsr_dacs = tdc_calibration_values
if not np.all(plsr_dacs == in_file_2.root.HitOrCalibration._v_attrs.scan_parameter_values):
raise NotImplementedError('The check calibration file has to have the same PlsrDAC values')
# Valid pixel have a calibration in the new and the old calibration
valid_pixel = np.where(~np.all((charge_calibration_1 == 0), axis=2) & ~np.all(np.isnan(charge_calibration_1), axis=2) & ~np.all((charge_calibration_2 == 0), axis=2) & ~np.all(np.isnan(charge_calibration_2), axis=2))
mean_charge_calibration = np.nanmean(charge_calibration_2[valid_pixel], axis=0)
offset_mean = np.nanmean((charge_calibration_2[valid_pixel] - charge_calibration_1[valid_pixel]), axis=0)
dPlsrDAC_dTDC = analysis_utils.smooth_differentiation(plsr_dacs, mean_charge_calibration, order=3, smoothness=0, derivation=1)
plt.clf()
plt.plot(plsr_dacs, offset_mean / dPlsrDAC_dTDC, '.-', label='PlsrDAC')
plt.plot(plsr_dacs, offset_mean, '.-', label='TDC')
plt.grid()
plt.xlabel('PlsrDAC')
plt.ylabel('Mean calibration offset')
plt.legend(loc=0)
plt.title('Mean offset between TDC calibration data, new - old ')
plt.savefig(filename_new_calibration[:-3] + '.pdf')
plt.show()
return offset_mean
def delete_disabled_regions(hits, enable_mask):
n_hits = hits.shape[0]
# Tread no hits case
if n_hits == 0:
return hits
# Column, row array with True for disabled pixels
disabled_region = ~enable_mask.astype(np.bool).T.copy()
n_disabled_pixels = np.count_nonzero(disabled_region)
# Extend disabled pixel mask by the neighbouring pixels
neighbour_pixels = [(-1, 0), (1, 0), (0, -1), (0, 1)] # Disable direct neighbouring pixels
for neighbour_pixel in neighbour_pixels:
disabled_region = np.logical_or(disabled_region, shift(disabled_region, shift=neighbour_pixel, cval=0))
logging.info('Masking %d additional pixel neighbouring %d disabled pixels', np.count_nonzero(disabled_region) - n_disabled_pixels, n_disabled_pixels)
# Make 1D selection array with disabled pixels
disabled_pixels = np.where(disabled_region)
disabled_pixels_1d = (disabled_pixels[0] + 1) * disabled_region.shape[1] + (disabled_pixels[1] + 1) # + 1 because pixel index 0,0 has column/row = 1
hits_1d = hits['column'].astype(np.uint32) * disabled_region.shape[1] + hits['row'] # change dtype to fit new number
hits = hits[np.in1d(hits_1d, disabled_pixels_1d, invert=True)]
logging.info('Lost %d hits (%d percent) due to disabling neighbours', n_hits - hits.shape[0], (1. - float(hits.shape[0]) / n_hits) * 100)
return hits
# Create data
with tb.open_file(input_file_hits, mode="r") as in_hit_file_h5:
cluster_hit_table = in_hit_file_h5.root.ClusterHits
try:
enabled_pixels = in_hit_file_h5.root.ClusterHits._v_attrs.enabled_pixels[:]
except AttributeError: # Old and simulate data do not have this info
logging.warning('No enabled pixel mask found in data! Assume all pixels are enabled.')
enabled_pixels = np.ones(shape=(336, 80))
# Result hists, initialized per condition
pixel_tdc_hists_per_condition = [np.zeros(shape=(80, 336, max_tdc), dtype=np.uint16) for _ in hit_selection_conditions] if hit_selection_conditions else []
pixel_tdc_timestamp_hists_per_condition = [np.zeros(shape=(80, 336, 256), dtype=np.uint16) for _ in hit_selection_conditions] if hit_selection_conditions else []
mean_pixel_tdc_hists_per_condition = [np.zeros(shape=(80, 336), dtype=np.uint16) for _ in hit_selection_conditions] if hit_selection_conditions else []
mean_pixel_tdc_timestamp_hists_per_condition = [np.zeros(shape=(80, 336), dtype=np.uint16) for _ in hit_selection_conditions] if hit_selection_conditions else []
tdc_hists_per_condition = [np.zeros(shape=(max_tdc), dtype=np.uint16) for _ in hit_selection_conditions] if hit_selection_conditions else []
tdc_corr_hists_per_condition = [np.zeros(shape=(max_tdc, 16), dtype=np.uint32) for _ in hit_selection_conditions] if hit_selection_conditions else []
n_hits_per_condition = [0 for _ in range(len(hit_selection_conditions) + 2)] # condition 1, 2 are all hits, hits of goode events
logging.info('Select hits and create TDC histograms for %d cut conditions', len(hit_selection_conditions))
progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=cluster_hit_table.shape[0], term_width=80)
progress_bar.start()
for cluster_hits, _ in analysis_utils.data_aligned_at_events(cluster_hit_table, chunk_size=10000000):
n_hits_per_condition[0] += cluster_hits.shape[0]
selected_events_cluster_hits = cluster_hits[np.logical_and(cluster_hits['TDC'] < max_tdc, (cluster_hits['event_status'] & event_status_select_mask) == event_status_condition)]
n_hits_per_condition[1] += selected_events_cluster_hits.shape[0]
for index, condition in enumerate(hit_selection_conditions):
selected_cluster_hits = analysis_utils.select_hits(selected_events_cluster_hits, condition)
if ignore_disabled_regions:
selected_cluster_hits = delete_disabled_regions(hits=selected_cluster_hits, enable_mask=enabled_pixels)
n_hits_per_condition[2 + index] += selected_cluster_hits.shape[0]
column, row, tdc = selected_cluster_hits['column'] - 1, selected_cluster_hits['row'] - 1, selected_cluster_hits['TDC']
pixel_tdc_hists_per_condition[index] += fast_analysis_utils.hist_3d_index(column, row, tdc, shape=(80, 336, max_tdc))
mean_pixel_tdc_hists_per_condition[index] = np.average(pixel_tdc_hists_per_condition[index], axis=2, weights=range(0, max_tdc)) * np.sum(np.arange(0, max_tdc)) / pixel_tdc_hists_per_condition[index].sum(axis=2)
tdc_timestamp = selected_cluster_hits['TDC_time_stamp']
pixel_tdc_timestamp_hists_per_condition[index] += fast_analysis_utils.hist_3d_index(column, row, tdc_timestamp, shape=(80, 336, 256))
mean_pixel_tdc_timestamp_hists_per_condition[index] = np.average(pixel_tdc_timestamp_hists_per_condition[index], axis=2, weights=range(0, 256)) * np.sum(np.arange(0, 256)) / pixel_tdc_timestamp_hists_per_condition[index].sum(axis=2)
tdc_hists_per_condition[index] = pixel_tdc_hists_per_condition[index].sum(axis=(0, 1))
tdc_corr_hists_per_condition[index] += fast_analysis_utils.hist_2d_index(tdc, selected_cluster_hits['tot'], shape=(max_tdc, 16))
progress_bar.update(n_hits_per_condition[0])
progress_bar.finish()
# Take TDC calibration if available and calculate charge for each TDC value and pixel
if calibration_file is not None:
with tb.open_file(calibration_file, mode="r") as in_file_calibration_h5:
tdc_calibration = in_file_calibration_h5.root.HitOrCalibration[:, :, :, 1]
tdc_calibration_values = in_file_calibration_h5.root.HitOrCalibration.attrs.scan_parameter_values[:]
if correct_calibration is not None:
tdc_calibration += get_calibration_correction(tdc_calibration, tdc_calibration_values, correct_calibration)
charge_calibration = get_charge(max_tdc, tdc_calibration_values, tdc_calibration)
else:
charge_calibration = None
# Store data of result histograms
with tb.open_file(input_file_hits[:-3] + '_tdc_hists.h5', mode="w") as out_file_h5:
for index, condition in enumerate(hit_selection_conditions):
pixel_tdc_hist_result = np.swapaxes(pixel_tdc_hists_per_condition[index], 0, 1)
pixel_tdc_timestamp_hist_result = np.swapaxes(pixel_tdc_timestamp_hists_per_condition[index], 0, 1)
mean_pixel_tdc_hist_result = np.swapaxes(mean_pixel_tdc_hists_per_condition[index], 0, 1)
mean_pixel_tdc_timestamp_hist_result = np.swapaxes(mean_pixel_tdc_timestamp_hists_per_condition[index], 0, 1)
tdc_hists_per_condition_result = tdc_hists_per_condition[index]
tdc_corr_hist_result = np.swapaxes(tdc_corr_hists_per_condition[index], 0, 1)
# Create result hists
out_1 = out_file_h5.create_carray(out_file_h5.root, name='HistPixelTdcCondition_%d' % index, title='Hist Pixel Tdc with %s' % condition, atom=tb.Atom.from_dtype(pixel_tdc_hist_result.dtype), shape=pixel_tdc_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_2 = out_file_h5.create_carray(out_file_h5.root, name='HistPixelTdcTimestampCondition_%d' % index, title='Hist Pixel Tdc Timestamp with %s' % condition, atom=tb.Atom.from_dtype(pixel_tdc_timestamp_hist_result.dtype), shape=pixel_tdc_timestamp_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_3 = out_file_h5.create_carray(out_file_h5.root, name='HistMeanPixelTdcCondition_%d' % index, title='Hist Mean Pixel Tdc with %s' % condition, atom=tb.Atom.from_dtype(mean_pixel_tdc_hist_result.dtype), shape=mean_pixel_tdc_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_4 = out_file_h5.create_carray(out_file_h5.root, name='HistMeanPixelTdcTimestampCondition_%d' % index, title='Hist Mean Pixel Tdc Timestamp with %s' % condition, atom=tb.Atom.from_dtype(mean_pixel_tdc_timestamp_hist_result.dtype), shape=mean_pixel_tdc_timestamp_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_5 = out_file_h5.create_carray(out_file_h5.root, name='HistTdcCondition_%d' % index, title='Hist Tdc with %s' % condition, atom=tb.Atom.from_dtype(tdc_hists_per_condition_result.dtype), shape=tdc_hists_per_condition_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_6 = out_file_h5.create_carray(out_file_h5.root, name='HistTdcCorrCondition_%d' % index, title='Hist Correlation Tdc/Tot with %s' % condition, atom=tb.Atom.from_dtype(tdc_corr_hist_result.dtype), shape=tdc_corr_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
# Add result hists information
out_1.attrs.dimensions, out_1.attrs.condition, out_1.attrs.tdc_values = 'column, row, TDC value', condition, range(max_tdc)
out_2.attrs.dimensions, out_2.attrs.condition, out_2.attrs.tdc_values = 'column, row, TDC time stamp value', condition, range(256)
out_3.attrs.dimensions, out_3.attrs.condition = 'column, row, mean TDC value', condition
out_4.attrs.dimensions, out_4.attrs.condition = 'column, row, mean TDC time stamp value', condition
out_5.attrs.dimensions, out_5.attrs.condition = 'PlsrDAC', condition
out_6.attrs.dimensions, out_6.attrs.condition = 'TDC, TOT', condition
out_1[:], out_2[:], out_3[:], out_4[:], out_5[:], out_6[:] = pixel_tdc_hist_result, pixel_tdc_timestamp_hist_result, mean_pixel_tdc_hist_result, mean_pixel_tdc_timestamp_hist_result, tdc_hists_per_condition_result, tdc_corr_hist_result
if charge_calibration is not None:
# Select only valid pixel for histogramming: they have data and a calibration (that is any charge(TDC) calibration != 0)
valid_pixel = np.where(np.logical_and(charge_calibration[:, :, :max_tdc].sum(axis=2) > 0, pixel_tdc_hist_result[:, :, :max_tdc].swapaxes(0, 1).sum(axis=2) > 0))
# Create charge histogram with mean TDC calibration
mean_charge_calibration = charge_calibration[valid_pixel][:, :max_tdc].mean(axis=0)
mean_tdc_hist = pixel_tdc_hist_result.swapaxes(0, 1)[valid_pixel][:, :max_tdc].mean(axis=0)
result_array = np.rec.array(np.column_stack((mean_charge_calibration, mean_tdc_hist)), dtype=[('charge', float), ('count', float)])
out_7 = out_file_h5.create_table(out_file_h5.root, name='HistMeanTdcCalibratedCondition_%d' % index, description=result_array.dtype, title='Hist Tdc with mean charge calibration and %s' % condition, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_7.attrs.condition = condition
out_7.attrs.n_pixel = valid_pixel[0].shape[0]
out_7.attrs.n_hits = pixel_tdc_hist_result.swapaxes(0, 1)[valid_pixel][:, :max_tdc].sum()
out_7.append(result_array)
# Create charge histogram with per pixel TDC calibration
x, y = charge_calibration[valid_pixel][:, :max_tdc].ravel(), np.ravel(pixel_tdc_hist_result.swapaxes(0, 1)[valid_pixel][:, :max_tdc].ravel())
y_hist, x_hist = y[x > 0], x[x > 0] # remove the hit tdcs without proper calibration plsrDAC(TDC) calibration
x, y, yerr = analysis_utils.get_profile_histogram(x_hist, y_hist, n_bins=n_bins)
result_array = np.rec.array(np.column_stack((x, y, yerr)), dtype=[('charge', float), ('count', float), ('count_error', float)])
out_8 = out_file_h5.create_table(out_file_h5.root, name='HistTdcCalibratedCondition_%d' % index, description=result_array.dtype, title='Hist Tdc with per pixel charge calibration and %s' % condition, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_8.attrs.condition = condition
out_8.attrs.n_pixel = valid_pixel[0].shape[0]
out_8.attrs.n_hits = y_hist.sum()
out_8.append(result_array)
# Plot Data
if plot_data:
with PdfPages(input_file_hits[:-3] + '_calibrated_tdc_hists.pdf') as output_pdf:
plot_hits_per_condition(output_pdf)
with tb.open_file(input_file_hits[:-3] + '_tdc_hists.h5', mode="r") as in_file_h5:
for node in in_file_h5.root: # go through the data and plot them
if 'MeanPixel' in node.name:
try:
plot_three_way(np.ma.masked_invalid(node[:]) * 1.5625, title='Mean TDC delay, hits with\n%s' % node._v_attrs.condition[:80] if 'Timestamp' in node.name else 'Mean TDC, hits with\n%s' % node._v_attrs.condition[:80], filename=output_pdf)
except ValueError:
logging.warning('Cannot plot TDC delay')
elif 'HistTdcCondition' in node.name:
hist_1d = node[:]
entry_index = np.where(hist_1d != 0)
if entry_index[0].shape[0] != 0:
max_index = np.amax(entry_index)
else:
max_index = max_tdc
plot_1d_hist(hist_1d[:max_index + 10], title='TDC histogram, hits with\n%s' % node._v_attrs.condition[:80] if 'Timestamp' not in node.name else 'TDC time stamp histogram, hits with\n%s' % node._v_attrs.condition[:80], x_axis_title='TDC' if 'Timestamp' not in node.name else 'TDC time stamp', filename=output_pdf)
elif 'HistPixelTdc' in node.name:
hist_3d = node[:]
entry_index = np.where(hist_3d.sum(axis=(0, 1)) != 0)
if entry_index[0].shape[0] != 0:
max_index = np.amax(entry_index)
else:
max_index = max_tdc
best_pixel_index = np.where(hist_3d.sum(axis=2) == np.amax(node[:].sum(axis=2)))
if best_pixel_index[0].shape[0] == 1: # there could be more than one pixel with most hits
try:
plot_1d_hist(hist_3d[best_pixel_index][0, :max_index], title='TDC histogram of pixel %d, %d\n%s' % (best_pixel_index[1] + 1, best_pixel_index[0] + 1, node._v_attrs.condition[:80]) if 'Timestamp' not in node.name else 'TDC time stamp histogram, hits of pixel %d, %d' % (best_pixel_index[1] + 1, best_pixel_index[0] + 1), x_axis_title='TDC' if 'Timestamp' not in node.name[:80] else 'TDC time stamp', filename=output_pdf)
except IndexError:
logging.warning('Cannot plot pixel TDC histogram')
elif 'HistTdcCalibratedCondition' in node.name:
plot_corrected_tdc_hist(node[:]['charge'], node[:]['count'], title='TDC histogram, %d pixel, per pixel TDC calib.\n%s' % (node._v_attrs.n_pixel, node._v_attrs.condition[:80]), output_pdf=output_pdf)
elif 'HistMeanTdcCalibratedCondition' in node.name:
plot_corrected_tdc_hist(node[:]['charge'], node[:]['count'], title='TDC histogram, %d pixel, mean TDC calib.\n%s' % (node._v_attrs.n_pixel, node._v_attrs.condition[:80]), output_pdf=output_pdf)
elif 'HistTdcCorr' in node.name:
plot_tdc_tot_correlation(node[:], node._v_attrs.condition, output_pdf)