def test_2d_index_histograming(self): # check compiled hist_2D_index function
x, y = np.random.randint(0, 100, 100), np.random.randint(0, 100, 100)
shape = (100, 100)
array_fast = analysis_utils.hist_2d_index(x, y, shape=shape)
array = np.histogram2d(x, y, bins=shape, range=[[0, shape[0]], [0, shape[1]]])[0]
shape = (5, 200) # shape that is too small for the indices to trigger exception
exception_ok = False
try:
array_fast = analysis_utils.hist_2d_index(x, y, shape=shape)
except IndexError:
exception_ok = True
except: # other exception that should not occur
pass
self.assertTrue(exception_ok & np.all(array == array_fast))
def test_2d_index_histograming(self): # check compiled hist_2D_index function
x, y = np.random.randint(0, 100, 100), np.random.randint(0, 100, 100)
shape = (100, 100)
array_fast = analysis_utils.hist_2d_index(x, y, shape=shape)
array = np.histogram2d(x, y, bins=shape, range=[[0, shape[0]], [0, shape[1]]])[0]
shape = (5, 200) # shape that is too small for the indices to trigger exception
exception_ok = False
try:
array_fast = analysis_utils.hist_2d_index(x, y, shape=shape)
except IndexError:
exception_ok = True
except: # other exception that should not occur
pass
self.assertTrue(exception_ok & np.all(array == array_fast))
def scan(self):
self.start_condition_triggered = False # set to true if the start condition is true once
self.stop_condition_triggered = False # set to true if the stop condition is true once
self.start_at = 0.01 # if more than start_at*activated_pixel see at least one hit the precise scanning is started
self.stop_at = 0.95 # if more than stop_at*activated_pixel see the maximum numbers of injection, the scan is stopped
self.record_data = False # set to true to activate data storage, so far not everything is recorded to ease data analysis
scan_parameter_range = [
0, (2**self.register.global_registers['PlsrDAC']['bitlength'])
]
if self.scan_parameters.PlsrDAC[0]:
scan_parameter_range[0] = self.scan_parameters.PlsrDAC[0]
if self.scan_parameters.PlsrDAC[1]:
scan_parameter_range[1] = self.scan_parameters.PlsrDAC[1]
logging.info(
"Scanning %s from %d to %d" %
('PlsrDAC', scan_parameter_range[0], scan_parameter_range[1]))
self.scan_parameter_value = scan_parameter_range[
0] # set to start value
self.search_distance = self.search_distance
self.data_points = 0 # counter variable to count the data points already recorded, have to be at least minimum_data_ponts
# calculate DCs to scan from the columns to ignore
enable_double_columns = range(0, 40)
if 1 in self.ignore_columns:
enable_double_columns.remove(0)
if set((78, 79, 80)).issubset(self.ignore_columns):
enable_double_columns.remove(39)
for double_column in range(1, 39):
if set((double_column * 2,
(double_column * 2) + 1)).issubset(self.ignore_columns):
enable_double_columns.remove(double_column)
logging.info("Use DCs: %s" % str(enable_double_columns))
self.select_arr_columns = range(0, 80)
for column in self.ignore_columns:
self.select_arr_columns.remove(column - 1)
while self.scan_parameter_value <= scan_parameter_range[
1]: # scan as long as scan parameter is smaller than defined maximum
if self.stop_run.is_set():
break
if self.record_data:
logging.info(
"Scan step %d (%s %d)" %
(self.data_points, 'PlsrDAC', self.scan_parameter_value))
commands = []
commands.extend(self.register.get_commands("ConfMode"))
self.register.set_global_register_value('PlsrDAC',
self.scan_parameter_value)
commands.extend(
self.register.get_commands("WrRegister", name=['PlsrDAC']))
self.register_utils.send_commands(commands)
with self.readout(PlsrDAC=self.scan_parameter_value):
cal_lvl1_command = self.register.get_commands(
"CAL")[0] + self.register.get_commands(
"zeros",
length=40)[0] + self.register.get_commands("LV1")[0]
scan_loop(self,
cal_lvl1_command,
repeat_command=self.n_injections,
use_delay=True,
mask_steps=self.mask_steps,
enable_mask_steps=self.enable_mask_steps,
enable_double_columns=enable_double_columns,
same_mask_for_all_dc=True,
eol_function=None,
digital_injection=False,
enable_shift_masks=self.enable_shift_masks,
disable_shift_masks=self.disable_shift_masks,
restore_shift_masks=False,
mask=invert_pixel_mask(
self.register.get_pixel_register_value('Enable'))
if self.use_enable_mask else None,
double_column_correction=self.pulser_dac_correction)
if not self.start_condition_triggered or self.data_points > self.minimum_data_points: # speed up, only create histograms when needed. Python is much too slow here.
if not self.start_condition_triggered and not self.record_data:
logging.info('Testing for start condition: %s %d' %
('PlsrDAC', self.scan_parameter_value))
if not self.stop_condition_triggered and self.record_data:
logging.info('Testing for stop condition: %s %d' %
('PlsrDAC', self.scan_parameter_value))
col, row = convert_data_array(
data_array_from_data_iterable(self.fifo_readout.data),
filter_func=is_data_record,
converter_func=get_col_row_array_from_data_record_array)
# using self written histogrammer in C++
occupancy_array = hist_2d_index(col - 1,
row - 1,
shape=(80, 336))
# using numpy
# occupancy_array = np.histogram2d(col, row, bins=(80, 336), range=[[1, 80], [1, 336]])[0]
self.scan_condition(occupancy_array)
# start condition is met for the first time
if self.start_condition_triggered and not self.record_data:
self.scan_parameter_value = self.scan_parameter_value - self.search_distance + self.step_size
if self.scan_parameter_value < 0:
self.scan_parameter_value = 0
logging.info('Starting threshold scan at %s %d' %
('PlsrDAC', self.scan_parameter_value))
self.scan_parameter_start = self.scan_parameter_value
self.record_data = True
continue
# saving data
if self.record_data:
self.data_points = self.data_points + 1
self.raw_data_file.append(
self.fifo_readout.data,
scan_parameters=self.scan_parameters._asdict())
# stop condition is met for the first time
if self.stop_condition_triggered and self.record_data:
logging.info('Stopping threshold scan at %s %d' %
('PlsrDAC', self.scan_parameter_value))
break
# increase scan parameter value
if not self.start_condition_triggered:
self.scan_parameter_value = self.scan_parameter_value + self.search_distance
else:
self.scan_parameter_value = self.scan_parameter_value + self.step_size
if self.scan_parameter_value >= scan_parameter_range[1]:
logging.warning(
"Reached maximum of PlsrDAC range... stopping scan")
def scan(self):
self.start_condition_triggered = False # set to true if the start condition is true once
self.stop_condition_triggered = False # set to true if the stop condition is true once
self.start_at = 0.01 # if more than start_at*activated_pixel see at least one hit the precise scanning is started
self.stop_at = 0.95 # if more than stop_at*activated_pixel see the maximum numbers of injection, the scan is stopped
self.record_data = False # set to true to activate data storage, so far not everything is recorded to ease data analysis
scan_parameter_range = [0, (2 ** self.register.global_registers['PlsrDAC']['bitlength'] - 1)]
if self.scan_parameters.PlsrDAC[0]:
scan_parameter_range[0] = self.scan_parameters.PlsrDAC[0]
if self.scan_parameters.PlsrDAC[1]:
scan_parameter_range[1] = self.scan_parameters.PlsrDAC[1]
logging.info("Scanning %s from %d to %d", 'PlsrDAC', scan_parameter_range[0], scan_parameter_range[1])
self.scan_parameter_value = scan_parameter_range[0] # set to start value
self.search_distance = self.search_distance
self.data_points = 0 # counter variable to count the data points already recorded, have to be at least minimum_data_ponts
# calculate DCs to scan from the columns to ignore
enable_double_columns = range(0, 40)
if 1 in self.ignore_columns:
enable_double_columns.remove(0)
if set((78, 79, 80)).issubset(self.ignore_columns):
enable_double_columns.remove(39)
for double_column in range(1, 39):
if set((double_column * 2, (double_column * 2) + 1)).issubset(self.ignore_columns):
enable_double_columns.remove(double_column)
logging.info("Use DCs: %s", str(enable_double_columns))
self.select_arr_columns = range(0, 80)
for column in self.ignore_columns:
self.select_arr_columns.remove(column - 1)
while self.scan_parameter_value <= scan_parameter_range[1]: # scan as long as scan parameter is smaller than defined maximum
if self.stop_run.is_set():
break
commands = []
commands.extend(self.register.get_commands("ConfMode"))
self.register.set_global_register_value('PlsrDAC', self.scan_parameter_value)
commands.extend(self.register.get_commands("WrRegister", name=['PlsrDAC']))
self.register_utils.send_commands(commands)
with self.readout(PlsrDAC=self.scan_parameter_value, reset_sram_fifo=True, fill_buffer=True, clear_buffer=True, callback=self.handle_data if self.record_data else None):
cal_lvl1_command = self.register.get_commands("CAL")[0] + self.register.get_commands("zeros", length=40)[0] + self.register.get_commands("LV1")[0]
scan_loop(self, cal_lvl1_command, repeat_command=self.n_injections, use_delay=True, mask_steps=self.mask_steps, enable_mask_steps=self.enable_mask_steps, enable_double_columns=enable_double_columns, same_mask_for_all_dc=True, eol_function=None, digital_injection=False, enable_shift_masks=self.enable_shift_masks, disable_shift_masks=self.disable_shift_masks, restore_shift_masks=False, mask=invert_pixel_mask(self.register.get_pixel_register_value('Enable')) if self.use_enable_mask else None, double_column_correction=self.pulser_dac_correction)
if not self.start_condition_triggered or self.data_points > self.minimum_data_points: # speed up, only create histograms when needed. Python is much too slow here.
if not self.start_condition_triggered and not self.record_data:
logging.info('Testing for start condition: %s %d', 'PlsrDAC', self.scan_parameter_value)
if not self.stop_condition_triggered and self.record_data:
logging.info('Testing for stop condition: %s %d', 'PlsrDAC', self.scan_parameter_value)
col, row = convert_data_array(data_array_from_data_iterable(self.fifo_readout.data), filter_func=is_data_record, converter_func=get_col_row_array_from_data_record_array)
if np.any(np.logical_and(col < 1, col > 80)) or np.any(np.logical_and(row < 1, row > 336)): # filter bad data records that can happen
logging.warning('There are undefined %d data records (e.g. random data)', np.count_nonzero(np.logical_and(col < 1, col > 80)) + np.count_nonzero(np.logical_and(row < 1, row > 336)))
col, row = col[np.logical_and(col > 0, col <= 80)], row[np.logical_and(row > 0, row < 336)]
occupancy_array = hist_2d_index(col - 1, row - 1, shape=(80, 336))
self.scan_condition(occupancy_array)
# start condition is met for the first time
if self.start_condition_triggered and not self.record_data:
self.scan_parameter_value = self.scan_parameter_value - self.search_distance + self.step_size
if self.scan_parameter_value < 0:
self.scan_parameter_value = 0
logging.info('Starting threshold scan at %s %d', 'PlsrDAC', self.scan_parameter_value)
self.scan_parameter_start = self.scan_parameter_value
self.record_data = True
continue
# saving data
if self.record_data:
self.data_points = self.data_points + 1
# stop condition is met for the first time
if self.stop_condition_triggered and self.record_data:
logging.info('Stopping threshold scan at %s %d', 'PlsrDAC', self.scan_parameter_value)
break
# increase scan parameter value
if not self.start_condition_triggered:
self.scan_parameter_value = self.scan_parameter_value + self.search_distance
else:
self.scan_parameter_value = self.scan_parameter_value + self.step_size
if self.scan_parameter_value >= scan_parameter_range[1]:
logging.warning("Reached maximum of PlsrDAC range... stopping scan")
def histogram_tdc_hits(input_file_hits, hit_selection_conditions, event_status_select_mask, event_status_condition, calibation_file=None, max_tdc=analysis_configuration['max_tdc'], n_bins=analysis_configuration['n_bins']):
for condition in hit_selection_conditions:
logging.info('Histogram tdc hits with %s', condition)
def get_charge(max_tdc, tdc_calibration_values, tdc_pixel_calibration): # return the charge from calibration
charge_calibration = np.zeros(shape=(80, 336, max_tdc))
for column in range(80):
for row in range(336):
actual_pixel_calibration = tdc_pixel_calibration[column, row, :]
if np.any(actual_pixel_calibration != 0) and np.all(np.isfinite(actual_pixel_calibration)):
interpolation = interp1d(x=actual_pixel_calibration, y=tdc_calibration_values, kind='slinear', bounds_error=False, fill_value=0)
charge_calibration[column, row, :] = interpolation(np.arange(max_tdc))
return charge_calibration
def plot_tdc_tot_correlation(data, condition, output_pdf):
logging.info('Plot correlation histogram for %s', condition)
plt.clf()
data = np.ma.array(data, mask=(data <= 0))
if np.ma.any(data > 0):
cmap = cm.get_cmap('jet', 200)
cmap.set_bad('w')
plt.title('Correlation with %s' % condition)
norm = colors.LogNorm()
z_max = data.max(fill_value=0)
plt.xlabel('TDC')
plt.ylabel('TOT')
im = plt.imshow(data, cmap=cmap, norm=norm, aspect='auto', interpolation='nearest') # , norm=norm)
divider = make_axes_locatable(plt.gca())
plt.gca().invert_yaxis()
cax = divider.append_axes("right", size="5%", pad=0.1)
plt.colorbar(im, cax=cax, ticks=np.linspace(start=0, stop=z_max, num=9, endpoint=True))
output_pdf.savefig()
else:
logging.warning('No data for correlation plotting for %s', condition)
def plot_hits_per_condition(output_pdf):
logging.info('Plot hits selection efficiency histogram for %d conditions', len(hit_selection_conditions) + 2)
labels = ['All Hits', 'Hits of\ngood events']
for condition in hit_selection_conditions:
condition = re.sub('[&]', '\n', condition)
condition = re.sub('[()]', '', condition)
labels.append(condition)
plt.bar(range(len(n_hits_per_condition)), n_hits_per_condition, align='center')
plt.xticks(range(len(n_hits_per_condition)), labels, size=8)
plt.title('Number of hits for different cuts')
plt.yscale('log')
plt.ylabel('#')
plt.grid()
for x, y in zip(np.arange(len(n_hits_per_condition)), n_hits_per_condition):
plt.annotate('%d' % (float(y) / float(n_hits_per_condition[0]) * 100.) + r'%', xy=(x, y / 2.), xycoords='data', color='grey', size=15)
output_pdf.savefig()
def plot_corrected_tdc_hist(x, y, title, output_pdf, point_style='-'):
logging.info('Plot TDC hist with TDC calibration')
plt.clf()
y /= np.amax(y) if y.shape[0] > 0 else y
plt.plot(x, y, point_style)
plt.title(title, size=10)
plt.xlabel('Charge [PlsrDAC]')
plt.ylabel('Count [a.u.]')
plt.grid()
output_pdf.savefig()
# Create data
with tb.openFile(input_file_hits, mode="r") as in_hit_file_h5:
cluster_hit_table = in_hit_file_h5.root.ClusterHits
# Result hists, initialized per condition
pixel_tdc_hists_per_condition = [np.zeros(shape=(80, 336, max_tdc), dtype=np.uint16) for _ in hit_selection_conditions] if hit_selection_conditions else []
pixel_tdc_timestamp_hists_per_condition = [np.zeros(shape=(80, 336, 256), dtype=np.uint16) for _ in hit_selection_conditions] if hit_selection_conditions else []
mean_pixel_tdc_hists_per_condition = [np.zeros(shape=(80, 336), dtype=np.uint16) for _ in hit_selection_conditions] if hit_selection_conditions else []
mean_pixel_tdc_timestamp_hists_per_condition = [np.zeros(shape=(80, 336), dtype=np.uint16) for _ in hit_selection_conditions] if hit_selection_conditions else []
tdc_hists_per_condition = [np.zeros(shape=(max_tdc), dtype=np.uint16) for _ in hit_selection_conditions] if hit_selection_conditions else []
tdc_corr_hists_per_condition = [np.zeros(shape=(max_tdc, 16), dtype=np.uint32) for _ in hit_selection_conditions] if hit_selection_conditions else []
n_hits_per_condition = [0 for _ in range(len(hit_selection_conditions) + 2)] # condition 1, 2 are all hits, hits of goode events
logging.info('Select hits and create TDC histograms for %d cut conditions', len(hit_selection_conditions))
progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=cluster_hit_table.shape[0], term_width=80)
progress_bar.start()
for cluster_hits, _ in analysis_utils.data_aligned_at_events(cluster_hit_table, chunk_size=1e8):
n_hits_per_condition[0] += cluster_hits.shape[0]
selected_events_cluster_hits = cluster_hits[np.logical_and(cluster_hits['TDC'] < max_tdc, (cluster_hits['event_status'] & event_status_select_mask) == event_status_condition)]
n_hits_per_condition[1] += selected_events_cluster_hits.shape[0]
for index, condition in enumerate(hit_selection_conditions):
selected_cluster_hits = analysis_utils.select_hits(selected_events_cluster_hits, condition)
n_hits_per_condition[2 + index] += selected_cluster_hits.shape[0]
column, row, tdc = selected_cluster_hits['column'] - 1, selected_cluster_hits['row'] - 1, selected_cluster_hits['TDC']
pixel_tdc_hists_per_condition[index] += analysis_utils.hist_3d_index(column, row, tdc, shape=(80, 336, max_tdc))
mean_pixel_tdc_hists_per_condition[index] = np.average(pixel_tdc_hists_per_condition[index], axis=2, weights=range(0, max_tdc)) * np.sum(np.arange(0, max_tdc)) / pixel_tdc_hists_per_condition[index].sum(axis=2)
tdc_timestamp = selected_cluster_hits['TDC_time_stamp']
pixel_tdc_timestamp_hists_per_condition[index] += analysis_utils.hist_3d_index(column, row, tdc_timestamp, shape=(80, 336, 256))
mean_pixel_tdc_timestamp_hists_per_condition[index] = np.average(pixel_tdc_timestamp_hists_per_condition[index], axis=2, weights=range(0, 256)) * np.sum(np.arange(0, 256)) / pixel_tdc_timestamp_hists_per_condition[index].sum(axis=2)
tdc_hists_per_condition[index] = pixel_tdc_hists_per_condition[index].sum(axis=(0, 1))
tdc_corr_hists_per_condition[index] += analysis_utils.hist_2d_index(tdc, selected_cluster_hits['tot'], shape=(max_tdc, 16))
progress_bar.update(n_hits_per_condition[0])
progress_bar.finish()
# Take TDC calibration if available and calculate charge for each TDC value and pixel
if calibation_file is not None:
with tb.openFile(calibation_file, mode="r") as in_file_calibration_h5:
tdc_calibration = in_file_calibration_h5.root.HitOrCalibration[:, :, :, 1]
tdc_calibration_values = in_file_calibration_h5.root.HitOrCalibration.attrs.scan_parameter_values[:]
charge_calibration = get_charge(max_tdc, tdc_calibration_values, tdc_calibration)
else:
charge_calibration = None
# Store data of result histograms
with tb.open_file(input_file_hits[:-3] + '_tdc_hists.h5', mode="w") as out_file_h5:
for index, condition in enumerate(hit_selection_conditions):
pixel_tdc_hist_result = np.swapaxes(pixel_tdc_hists_per_condition[index], 0, 1)
pixel_tdc_timestamp_hist_result = np.swapaxes(pixel_tdc_timestamp_hists_per_condition[index], 0, 1)
mean_pixel_tdc_hist_result = np.swapaxes(mean_pixel_tdc_hists_per_condition[index], 0, 1)
mean_pixel_tdc_timestamp_hist_result = np.swapaxes(mean_pixel_tdc_timestamp_hists_per_condition[index], 0, 1)
tdc_hists_per_condition_result = tdc_hists_per_condition[index]
tdc_corr_hist_result = np.swapaxes(tdc_corr_hists_per_condition[index], 0, 1)
# Create result hists
out_1 = out_file_h5.createCArray(out_file_h5.root, name='HistPixelTdcCondition_%d' % index, title='Hist Pixel Tdc with %s' % condition, atom=tb.Atom.from_dtype(pixel_tdc_hist_result.dtype), shape=pixel_tdc_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_2 = out_file_h5.createCArray(out_file_h5.root, name='HistPixelTdcTimestampCondition_%d' % index, title='Hist Pixel Tdc Timestamp with %s' % condition, atom=tb.Atom.from_dtype(pixel_tdc_timestamp_hist_result.dtype), shape=pixel_tdc_timestamp_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_3 = out_file_h5.createCArray(out_file_h5.root, name='HistMeanPixelTdcCondition_%d' % index, title='Hist Mean Pixel Tdc with %s' % condition, atom=tb.Atom.from_dtype(mean_pixel_tdc_hist_result.dtype), shape=mean_pixel_tdc_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_4 = out_file_h5.createCArray(out_file_h5.root, name='HistMeanPixelTdcTimestampCondition_%d' % index, title='Hist Mean Pixel Tdc Timestamp with %s' % condition, atom=tb.Atom.from_dtype(mean_pixel_tdc_timestamp_hist_result.dtype), shape=mean_pixel_tdc_timestamp_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_5 = out_file_h5.createCArray(out_file_h5.root, name='HistTdcCondition_%d' % index, title='Hist Tdc with %s' % condition, atom=tb.Atom.from_dtype(tdc_hists_per_condition_result.dtype), shape=tdc_hists_per_condition_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_6 = out_file_h5.createCArray(out_file_h5.root, name='HistTdcCorrCondition_%d' % index, title='Hist Correlation Tdc/Tot with %s' % condition, atom=tb.Atom.from_dtype(tdc_corr_hist_result.dtype), shape=tdc_corr_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
# Add result hists information
out_1.attrs.dimensions, out_1.attrs.condition, out_1.attrs.tdc_values = 'column, row, TDC value', condition, range(max_tdc)
out_2.attrs.dimensions, out_2.attrs.condition, out_2.attrs.tdc_values = 'column, row, TDC time stamp value', condition, range(256)
out_3.attrs.dimensions, out_3.attrs.condition = 'column, row, mean TDC value', condition
out_4.attrs.dimensions, out_4.attrs.condition = 'column, row, mean TDC time stamp value', condition
out_5.attrs.dimensions, out_5.attrs.condition = 'PlsrDAC', condition
out_6.attrs.dimensions, out_6.attrs.condition = 'TDC, TOT', condition
out_1[:], out_2[:], out_3[:], out_4[:], out_5[:], out_6[:] = pixel_tdc_hist_result, pixel_tdc_timestamp_hist_result, mean_pixel_tdc_hist_result, mean_pixel_tdc_timestamp_hist_result, tdc_hists_per_condition_result, tdc_corr_hist_result
if charge_calibration is not None:
# Select only valid pixel for histograming: they have data and a calibration (that is any charge(TDC) calibration != 0)
valid_pixel = np.where(np.logical_and(charge_calibration[:, :, :max_tdc].sum(axis=2) > 0, pixel_tdc_hist_result[:, :, :max_tdc].swapaxes(0, 1).sum(axis=2) > 0))
mean_charge_calibration = charge_calibration[valid_pixel][:, :max_tdc].mean(axis=0)
mean_tdc_hist = pixel_tdc_hist_result.swapaxes(0, 1)[valid_pixel][:, :max_tdc].mean(axis=0)
result_array = np.rec.array(np.column_stack((mean_charge_calibration, mean_tdc_hist)), dtype=[('charge', float), ('count', float)])
out_6 = out_file_h5.create_table(out_file_h5.root, name='HistMeanTdcCalibratedCondition_%d' % index, description=result_array.dtype, title='Hist Tdc with mean charge calibration and %s' % condition, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_6.attrs.condition = condition
out_6.attrs.n_pixel = valid_pixel[0].shape[0]
out_6.append(result_array)
# Create charge histogram with per pixel TDC(charge) calibration
x, y = charge_calibration[valid_pixel][:, :max_tdc].ravel(), np.ravel(pixel_tdc_hist_result.swapaxes(0, 1)[valid_pixel][:, :max_tdc].ravel())
y, x = y[x > 0], x[x > 0] # remove the hit tdcs without proper calibration plsrDAC(TDC) calibration
x, y, yerr = analysis_utils.get_profile_histogram(x, y, n_bins=n_bins)
result_array = np.rec.array(np.column_stack((x, y, yerr)), dtype=[('charge', float), ('count', float), ('count_error', float)])
out_7 = out_file_h5.create_table(out_file_h5.root, name='HistTdcCalibratedCondition_%d' % index, description=result_array.dtype, title='Hist Tdc with per pixel charge calibration and %s' % condition, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_7.attrs.condition = condition
out_7.attrs.n_pixel = valid_pixel[0].shape[0]
out_7.append(result_array)
# Plot Data
with PdfPages(input_file_hits[:-3] + '_calibrated_tdc_hists.pdf') as output_pdf:
plot_hits_per_condition(output_pdf)
with tb.open_file(input_file_hits[:-3] + '_tdc_hists.h5', mode="r") as in_file_h5:
for node in in_file_h5.root: # go through the data and plot them
if 'MeanPixel' in node.name:
try:
plot_three_way(np.ma.masked_invalid(node[:]) * 1.5625, title='Mean TDC delay, hits with\n%s' % node._v_attrs.condition if 'Timestamp' in node.name else 'Mean TDC, hits with\n%s' % node._v_attrs.condition, filename=output_pdf)
except ValueError:
logging.warning('Cannot plot TDC delay')
elif 'HistTdcCondition' in node.name:
hist_1d = node[:]
entry_index = np.where(hist_1d != 0)
if entry_index[0].shape[0] != 0:
max_index = np.amax(entry_index)
else:
max_index = max_tdc
plot_1d_hist(hist_1d[:max_index + 10], title='TDC histogram, hits with\n%s' % node._v_attrs.condition if 'Timestamp' not in node.name else 'TDC time stamp histogram, hits with\n%s' % node._v_attrs.condition, x_axis_title='TDC' if 'Timestamp' not in node.name else 'TDC time stamp', filename=output_pdf)
elif 'HistPixelTdc' in node.name:
hist_3d = node[:]
entry_index = np.where(hist_3d.sum(axis=(0, 1)) != 0)
if entry_index[0].shape[0] != 0:
max_index = np.amax(entry_index)
else:
max_index = max_tdc
best_pixel_index = np.where(hist_3d.sum(axis=2) == np.amax(node[:].sum(axis=2)))
if best_pixel_index[0].shape[0] == 1: # there could be more than one pixel with most hits
plot_1d_hist(hist_3d[best_pixel_index][0, :max_index], title='TDC histogram of pixel %d, %d\n%s' % (best_pixel_index[1] + 1, best_pixel_index[0] + 1, node._v_attrs.condition) if 'Timestamp' not in node.name else 'TDC time stamp histogram, hits of pixel %d, %d' % (best_pixel_index[1] + 1, best_pixel_index[0] + 1), x_axis_title='TDC' if 'Timestamp' not in node.name else 'TDC time stamp', filename=output_pdf)
elif 'HistTdcCalibratedCondition' in node.name:
plot_corrected_tdc_hist(node[:]['charge'], node[:]['count'], title='TDC histogram, %d pixel, per pixel TDC calib.\n%s' % (node._v_attrs.n_pixel, node._v_attrs.condition), output_pdf=output_pdf)
elif 'HistMeanTdcCalibratedCondition' in node.name:
plot_corrected_tdc_hist(node[:]['charge'], node[:]['count'], title='TDC histogram, %d pixel, mean TDC calib.\n%s' % (node._v_attrs.n_pixel, node._v_attrs.condition), output_pdf=output_pdf)
elif 'HistTdcCorr' in node.name:
plot_tdc_tot_correlation(node[:], node._v_attrs.condition, output_pdf)
def histogram_tdc_hits(input_file_hits, hit_selection_conditions, event_status_select_mask, event_status_condition, calibation_file=None, max_tdc=analysis_configuration['max_tdc'], n_bins=analysis_configuration['n_bins']):
for condition in hit_selection_conditions:
logging.info('Histogram tdc hits with %s', condition)
def get_charge(max_tdc, tdc_calibration_values, tdc_pixel_calibration): # return the charge from calibration
charge_calibration = np.zeros(shape=(80, 336, max_tdc))
for column in range(80):
for row in range(336):
actual_pixel_calibration = tdc_pixel_calibration[column, row, :]
if np.any(actual_pixel_calibration != 0) and np.all(np.isfinite(actual_pixel_calibration)):
interpolation = interp1d(x=actual_pixel_calibration, y=tdc_calibration_values, kind='slinear', bounds_error=False, fill_value=0)
charge_calibration[column, row, :] = interpolation(np.arange(max_tdc))
return charge_calibration
def plot_tdc_tot_correlation(data, condition, output_pdf):
logging.info('Plot correlation histogram for %s', condition)
plt.clf()
data = np.ma.array(data, mask=(data <= 0))
if np.ma.any(data > 0):
cmap = cm.get_cmap('jet', 200)
cmap.set_bad('w')
plt.title('Correlation with %s' % condition)
norm = colors.LogNorm()
z_max = data.max(fill_value=0)
plt.xlabel('TDC')
plt.ylabel('TOT')
im = plt.imshow(data, cmap=cmap, norm=norm, aspect='auto', interpolation='nearest') # , norm=norm)
divider = make_axes_locatable(plt.gca())
plt.gca().invert_yaxis()
cax = divider.append_axes("right", size="5%", pad=0.1)
plt.colorbar(im, cax=cax, ticks=np.linspace(start=0, stop=z_max, num=9, endpoint=True))
output_pdf.savefig()
else:
logging.warning('No data for correlation plotting for %s', condition)
def plot_hits_per_condition(output_pdf):
logging.info('Plot hits selection efficiency histogram for %d conditions', len(hit_selection_conditions) + 2)
labels = ['All Hits', 'Hits of\ngood events']
for condition in hit_selection_conditions:
condition = re.sub('[&]', '\n', condition)
condition = re.sub('[()]', '', condition)
labels.append(condition)
plt.bar(range(len(n_hits_per_condition)), n_hits_per_condition, align='center')
plt.xticks(range(len(n_hits_per_condition)), labels, size=8)
plt.title('Number of hits for different cuts')
plt.yscale('log')
plt.ylabel('#')
plt.grid()
for x, y in zip(np.arange(len(n_hits_per_condition)), n_hits_per_condition):
plt.annotate('%d' % (float(y) / float(n_hits_per_condition[0]) * 100.) + r'%', xy=(x, y / 2.), xycoords='data', color='grey', size=15)
output_pdf.savefig()
def plot_corrected_tdc_hist(x, y, title, output_pdf, point_style='-'):
logging.info('Plot TDC hist with TDC calibration')
plt.clf()
y /= np.amax(y) if y.shape[0] > 0 else y
plt.plot(x, y, point_style)
plt.title(title, size=10)
plt.xlabel('Charge [PlsrDAC]')
plt.ylabel('Count [a.u.]')
plt.grid()
output_pdf.savefig()
# Create data
with tb.openFile(input_file_hits, mode="r") as in_hit_file_h5:
cluster_hit_table = in_hit_file_h5.root.ClusterHits
# Result hists, initialized per condition
pixel_tdc_hists_per_condition = [np.zeros(shape=(80, 336, max_tdc), dtype=np.uint16) for _ in hit_selection_conditions] if hit_selection_conditions else []
pixel_tdc_timestamp_hists_per_condition = [np.zeros(shape=(80, 336, 256), dtype=np.uint16) for _ in hit_selection_conditions] if hit_selection_conditions else []
mean_pixel_tdc_hists_per_condition = [np.zeros(shape=(80, 336), dtype=np.uint16) for _ in hit_selection_conditions] if hit_selection_conditions else []
mean_pixel_tdc_timestamp_hists_per_condition = [np.zeros(shape=(80, 336), dtype=np.uint16) for _ in hit_selection_conditions] if hit_selection_conditions else []
tdc_hists_per_condition = [np.zeros(shape=(max_tdc), dtype=np.uint16) for _ in hit_selection_conditions] if hit_selection_conditions else []
tdc_corr_hists_per_condition = [np.zeros(shape=(max_tdc, 16), dtype=np.uint32) for _ in hit_selection_conditions] if hit_selection_conditions else []
n_hits_per_condition = [0 for _ in range(len(hit_selection_conditions) + 2)] # condition 1, 2 are all hits, hits of goode events
logging.info('Select hits and create TDC histograms for %d cut conditions', len(hit_selection_conditions))
progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=cluster_hit_table.shape[0], term_width=80)
progress_bar.start()
for cluster_hits, _ in analysis_utils.data_aligned_at_events(cluster_hit_table, chunk_size=1e8):
n_hits_per_condition[0] += cluster_hits.shape[0]
selected_events_cluster_hits = cluster_hits[np.logical_and(cluster_hits['TDC'] < max_tdc, (cluster_hits['event_status'] & event_status_select_mask) == event_status_condition)]
n_hits_per_condition[1] += selected_events_cluster_hits.shape[0]
for index, condition in enumerate(hit_selection_conditions):
selected_cluster_hits = analysis_utils.select_hits(selected_events_cluster_hits, condition)
n_hits_per_condition[2 + index] += selected_cluster_hits.shape[0]
column, row, tdc = selected_cluster_hits['column'] - 1, selected_cluster_hits['row'] - 1, selected_cluster_hits['TDC']
pixel_tdc_hists_per_condition[index] += analysis_utils.hist_3d_index(column, row, tdc, shape=(80, 336, max_tdc))
mean_pixel_tdc_hists_per_condition[index] = np.average(pixel_tdc_hists_per_condition[index], axis=2, weights=range(0, max_tdc)) * np.sum(np.arange(0, max_tdc)) / pixel_tdc_hists_per_condition[index].sum(axis=2)
tdc_timestamp = selected_cluster_hits['TDC_time_stamp']
pixel_tdc_timestamp_hists_per_condition[index] += analysis_utils.hist_3d_index(column, row, tdc_timestamp, shape=(80, 336, 256))
mean_pixel_tdc_timestamp_hists_per_condition[index] = np.average(pixel_tdc_timestamp_hists_per_condition[index], axis=2, weights=range(0, 256)) * np.sum(np.arange(0, 256)) / pixel_tdc_timestamp_hists_per_condition[index].sum(axis=2)
tdc_hists_per_condition[index] = pixel_tdc_hists_per_condition[index].sum(axis=(0, 1))
tdc_corr_hists_per_condition[index] += analysis_utils.hist_2d_index(tdc, selected_cluster_hits['tot'], shape=(max_tdc, 16))
progress_bar.update(n_hits_per_condition[0])
progress_bar.finish()
# Take TDC calibration if available and calculate charge for each TDC value and pixel
if calibation_file is not None:
with tb.openFile(calibation_file, mode="r") as in_file_calibration_h5:
tdc_calibration = in_file_calibration_h5.root.HitOrCalibration[:, :, :, 1]
tdc_calibration_values = in_file_calibration_h5.root.HitOrCalibration.attrs.scan_parameter_values[:]
charge_calibration = get_charge(max_tdc, tdc_calibration_values, tdc_calibration)
else:
charge_calibration = None
# Store data of result histograms
with tb.open_file(input_file_hits[:-3] + '_tdc_hists.h5', mode="w") as out_file_h5:
for index, condition in enumerate(hit_selection_conditions):
pixel_tdc_hist_result = np.swapaxes(pixel_tdc_hists_per_condition[index], 0, 1)
pixel_tdc_timestamp_hist_result = np.swapaxes(pixel_tdc_timestamp_hists_per_condition[index], 0, 1)
mean_pixel_tdc_hist_result = np.swapaxes(mean_pixel_tdc_hists_per_condition[index], 0, 1)
mean_pixel_tdc_timestamp_hist_result = np.swapaxes(mean_pixel_tdc_timestamp_hists_per_condition[index], 0, 1)
tdc_hists_per_condition_result = tdc_hists_per_condition[index]
tdc_corr_hist_result = np.swapaxes(tdc_corr_hists_per_condition[index], 0, 1)
# Create result hists
out_1 = out_file_h5.createCArray(out_file_h5.root, name='HistPixelTdcCondition_%d' % index, title='Hist Pixel Tdc with %s' % condition, atom=tb.Atom.from_dtype(pixel_tdc_hist_result.dtype), shape=pixel_tdc_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_2 = out_file_h5.createCArray(out_file_h5.root, name='HistPixelTdcTimestampCondition_%d' % index, title='Hist Pixel Tdc Timestamp with %s' % condition, atom=tb.Atom.from_dtype(pixel_tdc_timestamp_hist_result.dtype), shape=pixel_tdc_timestamp_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_3 = out_file_h5.createCArray(out_file_h5.root, name='HistMeanPixelTdcCondition_%d' % index, title='Hist Mean Pixel Tdc with %s' % condition, atom=tb.Atom.from_dtype(mean_pixel_tdc_hist_result.dtype), shape=mean_pixel_tdc_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_4 = out_file_h5.createCArray(out_file_h5.root, name='HistMeanPixelTdcTimestampCondition_%d' % index, title='Hist Mean Pixel Tdc Timestamp with %s' % condition, atom=tb.Atom.from_dtype(mean_pixel_tdc_timestamp_hist_result.dtype), shape=mean_pixel_tdc_timestamp_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_5 = out_file_h5.createCArray(out_file_h5.root, name='HistTdcCondition_%d' % index, title='Hist Tdc with %s' % condition, atom=tb.Atom.from_dtype(tdc_hists_per_condition_result.dtype), shape=tdc_hists_per_condition_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_6 = out_file_h5.createCArray(out_file_h5.root, name='HistTdcCorrCondition_%d' % index, title='Hist Correlation Tdc/Tot with %s' % condition, atom=tb.Atom.from_dtype(tdc_corr_hist_result.dtype), shape=tdc_corr_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
# Add result hists information
out_1.attrs.dimensions, out_1.attrs.condition, out_1.attrs.tdc_values = 'column, row, TDC value', condition, range(max_tdc)
out_2.attrs.dimensions, out_2.attrs.condition, out_2.attrs.tdc_values = 'column, row, TDC time stamp value', condition, range(256)
out_3.attrs.dimensions, out_3.attrs.condition = 'column, row, mean TDC value', condition
out_4.attrs.dimensions, out_4.attrs.condition = 'column, row, mean TDC time stamp value', condition
out_5.attrs.dimensions, out_5.attrs.condition = 'PlsrDAC', condition
out_6.attrs.dimensions, out_6.attrs.condition = 'TDC, TOT', condition
out_1[:], out_2[:], out_3[:], out_4[:], out_5[:], out_6[:] = pixel_tdc_hist_result, pixel_tdc_timestamp_hist_result, mean_pixel_tdc_hist_result, mean_pixel_tdc_timestamp_hist_result, tdc_hists_per_condition_result, tdc_corr_hist_result
if charge_calibration is not None:
# Select only valid pixel for histograming: they have data and a calibration (that is any charge(TDC) calibration != 0)
valid_pixel = np.where(np.logical_and(charge_calibration[:, :, :max_tdc].sum(axis=2) > 0, pixel_tdc_hist_result[:, :, :max_tdc].swapaxes(0, 1).sum(axis=2) > 0))
mean_charge_calibration = charge_calibration[valid_pixel][:, :max_tdc].mean(axis=0)
mean_tdc_hist = pixel_tdc_hist_result.swapaxes(0, 1)[valid_pixel][:, :max_tdc].mean(axis=0)
result_array = np.rec.array(np.column_stack((mean_charge_calibration, mean_tdc_hist)), dtype=[('charge', float), ('count', float)])
out_6 = out_file_h5.create_table(out_file_h5.root, name='HistMeanTdcCalibratedCondition_%d' % index, description=result_array.dtype, title='Hist Tdc with mean charge calibration and %s' % condition, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_6.attrs.condition = condition
out_6.attrs.n_pixel = valid_pixel[0].shape[0]
out_6.append(result_array)
# Create charge histogram with per pixel TDC(charge) calibration
x, y = charge_calibration[valid_pixel][:, :max_tdc].ravel(), np.ravel(pixel_tdc_hist_result.swapaxes(0, 1)[valid_pixel][:, :max_tdc].ravel())
y, x = y[x > 0], x[x > 0] # remove the hit tdcs without proper calibration plsrDAC(TDC) calibration
x, y, yerr = analysis_utils.get_profile_histogram(x, y, n_bins=n_bins)
result_array = np.rec.array(np.column_stack((x, y, yerr)), dtype=[('charge', float), ('count', float), ('count_error', float)])
out_7 = out_file_h5.create_table(out_file_h5.root, name='HistTdcCalibratedCondition_%d' % index, description=result_array.dtype, title='Hist Tdc with per pixel charge calibration and %s' % condition, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_7.attrs.condition = condition
out_7.attrs.n_pixel = valid_pixel[0].shape[0]
out_7.append(result_array)
# Plot Data
with PdfPages(input_file_hits[:-3] + '_calibrated_tdc_hists.pdf') as output_pdf:
plot_hits_per_condition(output_pdf)
with tb.open_file(input_file_hits[:-3] + '_tdc_hists.h5', mode="r") as in_file_h5:
for node in in_file_h5.root: # go through the data and plot them
if 'MeanPixel' in node.name:
try:
plotThreeWay(np.ma.masked_invalid(node[:]) * 1.5625, title='Mean TDC delay, hits with\n%s' % node._v_attrs.condition if 'Timestamp' in node.name else 'Mean TDC, hits with\n%s' % node._v_attrs.condition, filename=output_pdf)
except ValueError:
logging.warning('Cannot plot TDC delay')
elif 'HistTdcCondition' in node.name:
hist_1d = node[:]
entry_index = np.where(hist_1d != 0)
if entry_index[0].shape[0] != 0:
max_index = np.amax(entry_index)
else:
max_index = max_tdc
plot_1d_hist(hist_1d[:max_index + 10], title='TDC histogram, hits with\n%s' % node._v_attrs.condition if 'Timestamp' not in node.name else 'TDC time stamp histogram, hits with\n%s' % node._v_attrs.condition, x_axis_title='TDC' if 'Timestamp' not in node.name else 'TDC time stamp', filename=output_pdf)
elif 'HistPixelTdc' in node.name:
hist_3d = node[:]
entry_index = np.where(hist_3d.sum(axis=(0, 1)) != 0)
if entry_index[0].shape[0] != 0:
max_index = np.amax(entry_index)
else:
max_index = max_tdc
best_pixel_index = np.where(hist_3d.sum(axis=2) == np.amax(node[:].sum(axis=2)))
if best_pixel_index[0].shape[0] == 1: # there could be more than one pixel with most hits
plot_1d_hist(hist_3d[best_pixel_index][0, :max_index], title='TDC histogram of pixel %d, %d\n%s' % (best_pixel_index[1] + 1, best_pixel_index[0] + 1, node._v_attrs.condition) if 'Timestamp' not in node.name else 'TDC time stamp histogram, hits of pixel %d, %d' % (best_pixel_index[1] + 1, best_pixel_index[0] + 1), x_axis_title='TDC' if 'Timestamp' not in node.name else 'TDC time stamp', filename=output_pdf)
elif 'HistTdcCalibratedCondition' in node.name:
plot_corrected_tdc_hist(node[:]['charge'], node[:]['count'], title='TDC histogram, %d pixel, per pixel TDC calib.\n%s' % (node._v_attrs.n_pixel, node._v_attrs.condition), output_pdf=output_pdf)
elif 'HistMeanTdcCalibratedCondition' in node.name:
plot_corrected_tdc_hist(node[:]['charge'], node[:]['count'], title='TDC histogram, %d pixel, mean TDC calib.\n%s' % (node._v_attrs.n_pixel, node._v_attrs.condition), output_pdf=output_pdf)
elif 'HistTdcCorr' in node.name:
plot_tdc_tot_correlation(node[:], node._v_attrs.condition, output_pdf)
