def create_threshold_calibration(
scan_base_file_name,
create_plots=True
): # Create calibration function, can be called stand alone
def analyze_raw_data_file(file_name):
if os.path.isfile(file_name[:-3] +
'_interpreted.h5'): # skip analysis if already done
logging.warning('Analyzed data file ' + file_name +
' already exists. Skip analysis for this file.')
else:
with AnalyzeRawData(raw_data_file=file_name,
create_pdf=False) as analyze_raw_data:
analyze_raw_data.create_tot_hist = False
analyze_raw_data.create_tot_pixel_hist = False
analyze_raw_data.create_fitted_threshold_hists = True
analyze_raw_data.create_threshold_mask = True
analyze_raw_data.interpreter.set_warning_output(
False) # RX errors would fill the console
analyze_raw_data.interpret_word_table()
def store_calibration_data_as_table(out_file_h5,
mean_threshold_calibration,
mean_threshold_rms_calibration,
threshold_calibration,
parameter_values):
logging.info("Storing calibration data in a table...")
filter_table = tb.Filters(complib='blosc',
complevel=5,
fletcher32=False)
mean_threshold_calib_table = out_file_h5.createTable(
out_file_h5.root,
name='MeanThresholdCalibration',
description=data_struct.MeanThresholdCalibrationTable,
title='mean_threshold_calibration',
filters=filter_table)
threshold_calib_table = out_file_h5.createTable(
out_file_h5.root,
name='ThresholdCalibration',
description=data_struct.ThresholdCalibrationTable,
title='threshold_calibration',
filters=filter_table)
for column in range(80):
for row in range(336):
for parameter_value_index, parameter_value in enumerate(
parameter_values):
threshold_calib_table.row['column'] = column
threshold_calib_table.row['row'] = row
threshold_calib_table.row[
'parameter_value'] = parameter_value
threshold_calib_table.row[
'threshold'] = threshold_calibration[
column, row, parameter_value_index]
threshold_calib_table.row.append()
for parameter_value_index, parameter_value in enumerate(
parameter_values):
mean_threshold_calib_table.row['parameter_value'] = parameter_value
mean_threshold_calib_table.row[
'mean_threshold'] = mean_threshold_calibration[
parameter_value_index]
mean_threshold_calib_table.row[
'threshold_rms'] = mean_threshold_rms_calibration[
parameter_value_index]
mean_threshold_calib_table.row.append()
threshold_calib_table.flush()
mean_threshold_calib_table.flush()
logging.info("done")
def store_calibration_data_as_array(out_file_h5,
mean_threshold_calibration,
mean_threshold_rms_calibration,
threshold_calibration, parameter_name,
parameter_values):
logging.info("Storing calibration data in an array...")
filter_table = tb.Filters(complib='blosc',
complevel=5,
fletcher32=False)
mean_threshold_calib_array = out_file_h5.createCArray(
out_file_h5.root,
name='HistThresholdMeanCalibration',
atom=tb.Atom.from_dtype(mean_threshold_calibration.dtype),
shape=mean_threshold_calibration.shape,
title='mean_threshold_calibration',
filters=filter_table)
mean_threshold_calib_rms_array = out_file_h5.createCArray(
out_file_h5.root,
name='HistThresholdRMSCalibration',
atom=tb.Atom.from_dtype(mean_threshold_calibration.dtype),
shape=mean_threshold_calibration.shape,
title='mean_threshold_rms_calibration',
filters=filter_table)
threshold_calib_array = out_file_h5.createCArray(
out_file_h5.root,
name='HistThresholdCalibration',
atom=tb.Atom.from_dtype(threshold_calibration.dtype),
shape=threshold_calibration.shape,
title='threshold_calibration',
filters=filter_table)
mean_threshold_calib_array[:] = mean_threshold_calibration
mean_threshold_calib_rms_array[:] = mean_threshold_rms_calibration
threshold_calib_array[:] = threshold_calibration
mean_threshold_calib_array.attrs.dimensions = [
'column', 'row', parameter_name
]
mean_threshold_calib_rms_array.attrs.dimensions = [
'column', 'row', parameter_name
]
threshold_calib_array.attrs.dimensions = [
'column', 'row', parameter_name
]
mean_threshold_calib_array.attrs.scan_parameter_values = parameter_values
mean_threshold_calib_rms_array.attrs.scan_parameter_values = parameter_values
threshold_calib_array.attrs.scan_parameter_values = parameter_values
logging.info("done")
def mask_columns(pixel_array, ignore_columns):
idx = np.array(ignore_columns) - 1 # from FE to Array columns
m = np.zeros_like(pixel_array)
m[:, idx] = 1
return np.ma.masked_array(pixel_array, m)
raw_data_files = analysis_utils.get_data_file_names_from_scan_base(
scan_base_file_name,
filter_file_words=['interpreted', 'calibration_calibration'])
first_scan_base_file_name = scan_base_file_name if isinstance(
scan_base_file_name, basestring) else scan_base_file_name[
0] # multilpe scan_base_file_names for multiple runs
with tb.openFile(
first_scan_base_file_name + '.h5', mode="r"
) as in_file_h5: # deduce scan parameters from the first (and often only) scan base file name
ignore_columns = in_file_h5.root.configuration.run_conf[:][np.where(
in_file_h5.root.configuration.run_conf[:]['name'] ==
'ignore_columns')]['value'][0]
parameter_name = in_file_h5.root.configuration.run_conf[:][np.where(
in_file_h5.root.configuration.run_conf[:]['name'] ==
'scan_parameters')]['value'][0]
ignore_columns = ast.literal_eval(ignore_columns)
parameter_name = ast.literal_eval(parameter_name)[1][0]
calibration_file = first_scan_base_file_name + '_calibration'
for raw_data_file in raw_data_files: # analyze each raw data file, not using multithreading here, it is already used in s-curve fit
analyze_raw_data_file(raw_data_file)
files_per_parameter = analysis_utils.get_parameter_value_from_file_names(
[file_name[:-3] + '_interpreted.h5' for file_name in raw_data_files],
parameter_name,
unique=True,
sort=True)
logging.info("Create calibration from data")
mean_threshold_calibration = np.empty(shape=(len(raw_data_files), ),
dtype='<f8')
mean_threshold_rms_calibration = np.empty(shape=(len(raw_data_files), ),
dtype='<f8')
threshold_calibration = np.empty(shape=(80, 336, len(raw_data_files)),
dtype='<f8')
if create_plots:
logging.info('Saving calibration plots in: %s',
calibration_file + '.pdf')
output_pdf = PdfPages(calibration_file + '.pdf')
progress_bar = progressbar.ProgressBar(widgets=[
'',
progressbar.Percentage(), ' ',
progressbar.Bar(marker='*', left='|', right='|'), ' ',
progressbar.AdaptiveETA()
],
maxval=len(
files_per_parameter.items()),
term_width=80)
progress_bar.start()
parameter_values = []
for index, (analyzed_data_file,
parameters) in enumerate(files_per_parameter.items()):
parameter_values.append(parameters.values()[0][0])
with tb.openFile(analyzed_data_file, mode="r") as in_file_h5:
occupancy_masked = mask_columns(
pixel_array=in_file_h5.root.HistOcc[:],
ignore_columns=ignore_columns
) # mask the not scanned columns for analysis and plotting
thresholds_masked = mask_columns(
pixel_array=in_file_h5.root.HistThresholdFitted[:],
ignore_columns=ignore_columns)
if create_plots:
plot_three_way(hist=thresholds_masked,
title='Threshold Fitted for ' +
parameters.keys()[0] + ' = ' +
str(parameters.values()[0][0]),
filename=output_pdf)
plsr_dacs = analysis_utils.get_scan_parameter(
meta_data_array=in_file_h5.root.meta_data[:])['PlsrDAC']
plot_scurves(occupancy_hist=occupancy_masked,
scan_parameters=plsr_dacs,
scan_parameter_name='PlsrDAC',
filename=output_pdf)
# fill the calibration data arrays
mean_threshold_calibration[index] = np.ma.mean(thresholds_masked)
mean_threshold_rms_calibration[index] = np.ma.std(
thresholds_masked)
threshold_calibration[:, :, index] = thresholds_masked.T
progress_bar.update(index)
progress_bar.finish()
with tb.openFile(calibration_file + '.h5', mode="w") as out_file_h5:
store_calibration_data_as_array(
out_file_h5=out_file_h5,
mean_threshold_calibration=mean_threshold_calibration,
mean_threshold_rms_calibration=mean_threshold_rms_calibration,
threshold_calibration=threshold_calibration,
parameter_name=parameter_name,
parameter_values=parameter_values)
store_calibration_data_as_table(
out_file_h5=out_file_h5,
mean_threshold_calibration=mean_threshold_calibration,
mean_threshold_rms_calibration=mean_threshold_rms_calibration,
threshold_calibration=threshold_calibration,
parameter_values=parameter_values)
if create_plots:
plot_scatter(x=parameter_values,
y=mean_threshold_calibration,
title='Threshold calibration',
x_label=parameter_name,
y_label='Mean threshold',
log_x=False,
filename=output_pdf)
plot_scatter(x=parameter_values,
y=mean_threshold_calibration,
title='Threshold calibration',
x_label=parameter_name,
y_label='Mean threshold',
log_x=True,
filename=output_pdf)
output_pdf.close()
def analyze_injected_charge(data_analyzed_file):
logging.info('Analyze the injected charge')
with tb.openFile(data_analyzed_file, mode="r") as in_file_h5:
occupancy = in_file_h5.root.HistOcc[:].T
gdacs = analysis_utils.get_scan_parameter(in_file_h5.root.meta_data[:])['GDAC']
with PdfPages(data_analyzed_file[:-3] + '.pdf') as plot_file:
plotting.plot_scatter(gdacs, occupancy.sum(axis=(0, 1)), title='Single pixel hit rate at different thresholds', x_label='Threshold setting [GDAC]', y_label='Single pixel hit rate', log_x=True, filename=plot_file)
if analysis_configuration['input_file_calibration']:
with tb.openFile(analysis_configuration['input_file_calibration'], mode="r") as in_file_calibration_h5: # read calibration file from calibrate_threshold_gdac scan
mean_threshold_calibration = in_file_calibration_h5.root.MeanThresholdCalibration[:]
threshold_calibration_array = in_file_calibration_h5.root.HistThresholdCalibration[:]
gdac_range_calibration = np.array(in_file_calibration_h5.root.HistThresholdCalibration._v_attrs.scan_parameter_values)
gdac_range_source_scan = gdacs
# Select data that is within the given GDAC range, (min_gdac, max_gdac)
sel = np.where(np.logical_and(gdac_range_source_scan >= analysis_configuration['min_gdac'], gdac_range_source_scan <= analysis_configuration['max_gdac']))[0]
gdac_range_source_scan = gdac_range_source_scan[sel]
occupancy = occupancy[:, :, sel]
sel = np.where(np.logical_and(gdac_range_calibration >= analysis_configuration['min_gdac'], gdac_range_calibration <= analysis_configuration['max_gdac']))[0]
gdac_range_calibration = gdac_range_calibration[sel]
threshold_calibration_array = threshold_calibration_array[:, :, sel]
logging.info('Analyzing source scan data with %d GDAC settings from %d to %d with minimum step sizes from %d to %d', len(gdac_range_source_scan), np.min(gdac_range_source_scan), np.max(gdac_range_source_scan), np.min(np.gradient(gdac_range_source_scan)), np.max(np.gradient(gdac_range_source_scan)))
logging.info('Use calibration data with %d GDAC settings from %d to %d with minimum step sizes from %d to %d', len(gdac_range_calibration), np.min(gdac_range_calibration), np.max(gdac_range_calibration), np.min(np.gradient(gdac_range_calibration)), np.max(np.gradient(gdac_range_calibration)))
# rate_normalization of the total hit number for each GDAC setting
rate_normalization = 1.
if analysis_configuration['normalize_rate']:
rate_normalization = analysis_utils.get_rate_normalization(hit_file=hit_file, cluster_file=hit_file, parameter='GDAC', reference=analysis_configuration['normalization_reference'], plot=analysis_configuration['plot_normalization'])
# correcting the hit numbers for the different cluster sizes
correction_factors = 1.
if analysis_configuration['use_cluster_rate_correction']:
correction_h5 = tb.openFile(cluster_sizes_file, mode="r")
cluster_size_histogram = correction_h5.root.AllHistClusterSize[:]
correction_factors = analysis_utils.get_hit_rate_correction(gdacs=gdac_range_source_scan, calibration_gdacs=gdac_range_source_scan, cluster_size_histogram=cluster_size_histogram)
if analysis_configuration['plot_cluster_sizes']:
plot_cluster_sizes(correction_h5, in_file_calibration_h5, gdac_range=gdac_range_source_scan)
pixel_thresholds = analysis_utils.get_pixel_thresholds_from_calibration_array(gdacs=gdac_range_source_scan, calibration_gdacs=gdac_range_calibration, threshold_calibration_array=threshold_calibration_array) # interpolates the threshold at the source scan GDAC setting from the calibration
pixel_hits = occupancy # create hit array with shape (col, row, ...)
pixel_hits = pixel_hits * correction_factors * rate_normalization
# choose region with pixels that have a sufficient occupancy but are not too hot
good_pixel = analysis_utils.select_good_pixel_region(pixel_hits, col_span=analysis_configuration['col_span'], row_span=analysis_configuration['row_span'], min_cut_threshold=analysis_configuration['min_cut_threshold'], max_cut_threshold=analysis_configuration['max_cut_threshold'])
pixel_mask = ~np.ma.getmaskarray(good_pixel)
selected_pixel_hits = pixel_hits[pixel_mask, :] # reduce the data to pixels that are in the good pixel region
selected_pixel_thresholds = pixel_thresholds[pixel_mask, :] # reduce the data to pixels that are in the good pixel region
plotting.plot_occupancy(good_pixel.T, title='Selected pixel for analysis (' + str(len(selected_pixel_hits)) + ')', filename=plot_file)
# reshape to one dimension
x = selected_pixel_thresholds.flatten()
y = selected_pixel_hits.flatten()
# nothing should be NAN/INF, NAN/INF is not supported yet
if np.isfinite(x).shape != x.shape or np.isfinite(y).shape != y.shape:
logging.warning('There are pixels with NaN or INF threshold or hit values, analysis will fail')
# calculated profile histogram
x_p, y_p, y_p_e = analysis_utils.get_profile_histogram(x, y, n_bins=analysis_configuration['n_bins']) # profile histogram data
# select only the data point where the calibration worked
selected_data = np.logical_and(x_p > analysis_configuration['min_thr'] / analysis_configuration['vcal_calibration'], x_p < analysis_configuration['max_thr'] / analysis_configuration['vcal_calibration'])
x_p = x_p[selected_data]
y_p = y_p[selected_data]
y_p_e = y_p_e[selected_data]
if len(y_p_e[y_p_e == 0]) != 0:
logging.warning('There are bins without any data, guessing the error bars')
y_p_e[y_p_e == 0] = np.amin(y_p_e[y_p_e != 0])
smoothed_data = analysis_utils.smooth_differentiation(x_p, y_p, weigths=1 / y_p_e, order=3, smoothness=analysis_configuration['smoothness'], derivation=0)
smoothed_data_diff = analysis_utils.smooth_differentiation(x_p, y_p, weigths=1 / y_p_e, order=3, smoothness=analysis_configuration['smoothness'], derivation=1)
with tb.openFile(data_analyzed_file[:-3] + '_result.h5', mode="w") as out_file_h5:
result_1 = np.rec.array(np.column_stack((x_p, y_p, y_p_e)), dtype=[('charge', float), ('count', float), ('count_error', float)])
result_2 = np.rec.array(np.column_stack((x_p, smoothed_data)), dtype=[('charge', float), ('count', float)])
result_3 = np.rec.array(np.column_stack((x_p, -smoothed_data_diff)), dtype=[('charge', float), ('count', float)])
out_1 = out_file_h5.create_table(out_file_h5.root, name='ProfileHistogram', description=result_1.dtype, title='Single pixel count rate combined with a profile histogram', filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_2 = out_file_h5.create_table(out_file_h5.root, name='ProfileHistogramSpline', description=result_2.dtype, title='Single pixel count rate combined with a profile histogram and spline smoothed', filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
out_3 = out_file_h5.create_table(out_file_h5.root, name='ChargeHistogram', description=result_3.dtype, title='Charge histogram with threshold method and per pixel calibration', filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
for key, value in analysis_configuration.iteritems():
out_1.attrs[key] = value
out_2.attrs[key] = value
out_3.attrs[key] = value
out_1.append(result_1)
out_2.append(result_2)
out_3.append(result_3)
plot_result(x_p, y_p, y_p_e, smoothed_data, smoothed_data_diff)
# calculate and plot mean results
x_mean = analysis_utils.get_mean_threshold_from_calibration(gdac_range_source_scan, mean_threshold_calibration)
y_mean = selected_pixel_hits.mean(axis=(0))
plotting.plot_scatter(np.array(gdac_range_source_scan), y_mean, log_x=True, plot_range=None, title='Mean single pixel cluster rate at different thresholds', x_label='threshold setting [GDAC]', y_label='mean single pixel cluster rate', filename=plot_file)
plotting.plot_scatter(x_mean * analysis_configuration['vcal_calibration'], y_mean, plot_range=(analysis_configuration['min_thr'], analysis_configuration['max_thr']), title='Mean single pixel cluster rate at different thresholds', x_label='mean threshold [e]', y_label='mean single pixel cluster rate', filename=plot_file)
if analysis_configuration['use_cluster_rate_correction']:
correction_h5.close()
def analyze_injected_charge(data_analyzed_file):
logging.info('Analyze the injected charge')
with tb.openFile(data_analyzed_file, mode="r") as in_file_h5:
occupancy = in_file_h5.root.HistOcc[:].T
gdacs = analysis_utils.get_scan_parameter(
in_file_h5.root.meta_data[:])['GDAC']
with PdfPages(data_analyzed_file[:-3] + '.pdf') as plot_file:
plotting.plot_scatter(
gdacs,
occupancy.sum(axis=(0, 1)),
title='Single pixel hit rate at different thresholds',
x_label='Threshold setting [GDAC]',
y_label='Single pixel hit rate',
log_x=True,
filename=plot_file)
if analysis_configuration['input_file_calibration']:
with tb.openFile(
analysis_configuration['input_file_calibration'],
mode="r"
) as in_file_calibration_h5: # read calibration file from calibrate_threshold_gdac scan
mean_threshold_calibration = in_file_calibration_h5.root.MeanThresholdCalibration[:]
threshold_calibration_array = in_file_calibration_h5.root.HistThresholdCalibration[:]
gdac_range_calibration = np.array(
in_file_calibration_h5.root.HistThresholdCalibration.
_v_attrs.scan_parameter_values)
gdac_range_source_scan = gdacs
# Select data that is within the given GDAC range, (min_gdac, max_gdac)
sel = np.where(
np.logical_and(
gdac_range_source_scan >=
analysis_configuration['min_gdac'],
gdac_range_source_scan <=
analysis_configuration['max_gdac']))[0]
gdac_range_source_scan = gdac_range_source_scan[sel]
occupancy = occupancy[:, :, sel]
sel = np.where(
np.logical_and(
gdac_range_calibration >=
analysis_configuration['min_gdac'],
gdac_range_calibration <=
analysis_configuration['max_gdac']))[0]
gdac_range_calibration = gdac_range_calibration[sel]
threshold_calibration_array = threshold_calibration_array[:, :,
sel]
logging.info(
'Analyzing source scan data with %d GDAC settings from %d to %d with minimum step sizes from %d to %d',
len(gdac_range_source_scan),
np.min(gdac_range_source_scan),
np.max(gdac_range_source_scan),
np.min(np.gradient(gdac_range_source_scan)),
np.max(np.gradient(gdac_range_source_scan)))
logging.info(
'Use calibration data with %d GDAC settings from %d to %d with minimum step sizes from %d to %d',
len(gdac_range_calibration),
np.min(gdac_range_calibration),
np.max(gdac_range_calibration),
np.min(np.gradient(gdac_range_calibration)),
np.max(np.gradient(gdac_range_calibration)))
# rate_normalization of the total hit number for each GDAC setting
rate_normalization = 1.
if analysis_configuration['normalize_rate']:
rate_normalization = analysis_utils.get_rate_normalization(
hit_file=hit_file,
cluster_file=hit_file,
parameter='GDAC',
reference=analysis_configuration[
'normalization_reference'],
plot=analysis_configuration['plot_normalization'])
# correcting the hit numbers for the different cluster sizes
correction_factors = 1.
if analysis_configuration['use_cluster_rate_correction']:
correction_h5 = tb.openFile(cluster_sizes_file,
mode="r")
cluster_size_histogram = correction_h5.root.AllHistClusterSize[:]
correction_factors = analysis_utils.get_hit_rate_correction(
gdacs=gdac_range_source_scan,
calibration_gdacs=gdac_range_source_scan,
cluster_size_histogram=cluster_size_histogram)
if analysis_configuration['plot_cluster_sizes']:
plot_cluster_sizes(
correction_h5,
in_file_calibration_h5,
gdac_range=gdac_range_source_scan)
pixel_thresholds = analysis_utils.get_pixel_thresholds_from_calibration_array(
gdacs=gdac_range_source_scan,
calibration_gdacs=gdac_range_calibration,
threshold_calibration_array=threshold_calibration_array
) # interpolates the threshold at the source scan GDAC setting from the calibration
pixel_hits = occupancy # create hit array with shape (col, row, ...)
pixel_hits = pixel_hits * correction_factors * rate_normalization
# choose region with pixels that have a sufficient occupancy but are not too hot
good_pixel = analysis_utils.select_good_pixel_region(
pixel_hits,
col_span=analysis_configuration['col_span'],
row_span=analysis_configuration['row_span'],
min_cut_threshold=analysis_configuration[
'min_cut_threshold'],
max_cut_threshold=analysis_configuration[
'max_cut_threshold'])
pixel_mask = ~np.ma.getmaskarray(good_pixel)
selected_pixel_hits = pixel_hits[
pixel_mask, :] # reduce the data to pixels that are in the good pixel region
selected_pixel_thresholds = pixel_thresholds[
pixel_mask, :] # reduce the data to pixels that are in the good pixel region
plotting.plot_occupancy(
good_pixel.T,
title='Selected pixel for analysis (' +
str(len(selected_pixel_hits)) + ')',
filename=plot_file)
# reshape to one dimension
x = selected_pixel_thresholds.flatten()
y = selected_pixel_hits.flatten()
# nothing should be NAN/INF, NAN/INF is not supported yet
if np.isfinite(x).shape != x.shape or np.isfinite(
y).shape != y.shape:
logging.warning(
'There are pixels with NaN or INF threshold or hit values, analysis will fail'
)
# calculated profile histogram
x_p, y_p, y_p_e = analysis_utils.get_profile_histogram(
x, y, n_bins=analysis_configuration['n_bins']
) # profile histogram data
# select only the data point where the calibration worked
selected_data = np.logical_and(
x_p > analysis_configuration['min_thr'] /
analysis_configuration['vcal_calibration'],
x_p < analysis_configuration['max_thr'] /
analysis_configuration['vcal_calibration'])
x_p = x_p[selected_data]
y_p = y_p[selected_data]
y_p_e = y_p_e[selected_data]
if len(y_p_e[y_p_e == 0]) != 0:
logging.warning(
'There are bins without any data, guessing the error bars'
)
y_p_e[y_p_e == 0] = np.amin(y_p_e[y_p_e != 0])
smoothed_data = analysis_utils.smooth_differentiation(
x_p,
y_p,
weigths=1 / y_p_e,
order=3,
smoothness=analysis_configuration['smoothness'],
derivation=0)
smoothed_data_diff = analysis_utils.smooth_differentiation(
x_p,
y_p,
weigths=1 / y_p_e,
order=3,
smoothness=analysis_configuration['smoothness'],
derivation=1)
with tb.openFile(data_analyzed_file[:-3] + '_result.h5',
mode="w") as out_file_h5:
result_1 = np.rec.array(np.column_stack(
(x_p, y_p, y_p_e)),
dtype=[('charge', float),
('count', float),
('count_error', float)])
result_2 = np.rec.array(np.column_stack(
(x_p, smoothed_data)),
dtype=[('charge', float),
('count', float)])
result_3 = np.rec.array(np.column_stack(
(x_p, -smoothed_data_diff)),
dtype=[('charge', float),
('count', float)])
out_1 = out_file_h5.create_table(
out_file_h5.root,
name='ProfileHistogram',
description=result_1.dtype,
title=
'Single pixel count rate combined with a profile histogram',
filters=tb.Filters(complib='blosc',
complevel=5,
fletcher32=False))
out_2 = out_file_h5.create_table(
out_file_h5.root,
name='ProfileHistogramSpline',
description=result_2.dtype,
title=
'Single pixel count rate combined with a profile histogram and spline smoothed',
filters=tb.Filters(complib='blosc',
complevel=5,
fletcher32=False))
out_3 = out_file_h5.create_table(
out_file_h5.root,
name='ChargeHistogram',
description=result_3.dtype,
title=
'Charge histogram with threshold method and per pixel calibration',
filters=tb.Filters(complib='blosc',
complevel=5,
fletcher32=False))
for key, value in analysis_configuration.iteritems():
out_1.attrs[key] = value
out_2.attrs[key] = value
out_3.attrs[key] = value
out_1.append(result_1)
out_2.append(result_2)
out_3.append(result_3)
plot_result(x_p, y_p, y_p_e, smoothed_data,
smoothed_data_diff)
# calculate and plot mean results
x_mean = analysis_utils.get_mean_threshold_from_calibration(
gdac_range_source_scan, mean_threshold_calibration)
y_mean = selected_pixel_hits.mean(axis=(0))
plotting.plot_scatter(
np.array(gdac_range_source_scan),
y_mean,
log_x=True,
plot_range=None,
title=
'Mean single pixel cluster rate at different thresholds',
x_label='threshold setting [GDAC]',
y_label='mean single pixel cluster rate',
filename=plot_file)
plotting.plot_scatter(
x_mean * analysis_configuration['vcal_calibration'],
y_mean,
plot_range=(analysis_configuration['min_thr'],
analysis_configuration['max_thr']),
title=
'Mean single pixel cluster rate at different thresholds',
x_label='mean threshold [e]',
y_label='mean single pixel cluster rate',
filename=plot_file)
if analysis_configuration['use_cluster_rate_correction']:
correction_h5.close()
def analyse_n_cluster_per_event(scan_base,
include_no_cluster=False,
time_line_absolute=True,
combine_n_readouts=1000,
chunk_size=10000000,
plot_n_cluster_hists=False,
output_pdf=None,
output_file=None):
''' Determines the number of cluster per event as a function of time. Therefore the data of a fixed number of read outs are combined ('combine_n_readouts').
Parameters
----------
scan_base: list of str
scan base names (e.g.: ['//data//SCC_50_fei4_self_trigger_scan_390', ]
include_no_cluster: bool
Set to true to also consider all events without any hit.
combine_n_readouts: int
the number of read outs to combine (e.g. 1000)
max_chunk_size: int
the maximum chunk size used during read, if too big memory error occurs, if too small analysis takes longer
output_pdf: PdfPages
PdfPages file object, if none the plot is printed to screen
'''
time_stamp = []
n_cluster = []
start_time_set = False
for data_file in scan_base:
with tb.open_file(data_file + '_interpreted.h5',
mode="r+") as in_cluster_file_h5:
# get data and data pointer
meta_data_array = in_cluster_file_h5.root.meta_data[:]
cluster_table = in_cluster_file_h5.root.Cluster
# determine the event ranges to analyze (timestamp_start, start_event_number, stop_event_number)
parameter_ranges = np.column_stack(
(analysis_utils.get_ranges_from_array(
meta_data_array['timestamp_start'][::combine_n_readouts]),
analysis_utils.get_ranges_from_array(
meta_data_array['event_number'][::combine_n_readouts])))
# create a event_numer index (important for speed)
analysis_utils.index_event_number(cluster_table)
# initialize the analysis and set settings
analyze_data = AnalyzeRawData()
analyze_data.create_tot_hist = False
analyze_data.create_bcid_hist = False
# variables for read speed up
index = 0 # index where to start the read out, 0 at the beginning, increased during looping
best_chunk_size = chunk_size
total_cluster = cluster_table.shape[0]
progress_bar = progressbar.ProgressBar(widgets=[
'',
progressbar.Percentage(), ' ',
progressbar.Bar(marker='*', left='|', right='|'), ' ',
progressbar.AdaptiveETA()
],
maxval=total_cluster,
term_width=80)
progress_bar.start()
# loop over the selected events
for parameter_index, parameter_range in enumerate(
parameter_ranges):
logging.debug('Analyze time stamp ' + str(parameter_range[0]) +
' and data from events = [' +
str(parameter_range[2]) + ',' +
str(parameter_range[3]) + '[ ' + str(
int(
float(
float(parameter_index) /
float(len(parameter_ranges)) *
100.0))) + '%')
analyze_data.reset() # resets the data of the last analysis
# loop over the cluster in the actual selected events with optimizations: determine best chunk size, start word index given
readout_cluster_len = 0 # variable to calculate a optimal chunk size value from the number of hits for speed up
hist = None
for clusters, index in analysis_utils.data_aligned_at_events(
cluster_table,
start_event_number=parameter_range[2],
stop_event_number=parameter_range[3],
start_index=index,
chunk_size=best_chunk_size):
n_cluster_per_event = analysis_utils.get_n_cluster_in_events(
clusters['event_number']
)[:,
1] # array with the number of cluster per event, cluster per event are at least 1
if hist is None:
hist = np.histogram(n_cluster_per_event,
bins=10,
range=(0, 10))[0]
else:
hist = np.add(
hist,
np.histogram(n_cluster_per_event,
bins=10,
range=(0, 10))[0])
if include_no_cluster and parameter_range[
3] is not None: # happend for the last readout
hist[0] = (parameter_range[3] -
parameter_range[2]) - len(
n_cluster_per_event
) # add the events without any cluster
readout_cluster_len += clusters.shape[0]
total_cluster -= len(clusters)
progress_bar.update(index)
best_chunk_size = int(1.5 * readout_cluster_len) if int(
1.05 * readout_cluster_len
) < chunk_size else chunk_size # to increase the readout speed, estimated the number of hits for one read instruction
if plot_n_cluster_hists:
plotting.plot_1d_hist(
hist,
title='Number of cluster per event at ' +
str(parameter_range[0]),
x_axis_title='Number of cluster',
y_axis_title='#',
log_y=True,
filename=output_pdf)
hist = hist.astype('f4') / np.sum(
hist) # calculate fraction from total numbers
if time_line_absolute:
time_stamp.append(parameter_range[0])
else:
if not start_time_set:
start_time = parameter_ranges[0, 0]
start_time_set = True
time_stamp.append((parameter_range[0] - start_time) / 60.0)
n_cluster.append(hist)
progress_bar.finish()
if total_cluster != 0:
logging.warning(
'Not all clusters were selected during analysis. Analysis is therefore not exact'
)
if time_line_absolute:
plotting.plot_scatter_time(
time_stamp,
n_cluster,
title='Number of cluster per event as a function of time',
marker_style='o',
filename=output_pdf,
legend=('0 cluster', '1 cluster', '2 cluster',
'3 cluster') if include_no_cluster else
('0 cluster not plotted', '1 cluster', '2 cluster', '3 cluster'))
else:
plotting.plot_scatter(
time_stamp,
n_cluster,
title='Number of cluster per event as a function of time',
x_label='time [min.]',
marker_style='o',
filename=output_pdf,
legend=('0 cluster', '1 cluster', '2 cluster',
'3 cluster') if include_no_cluster else
('0 cluster not plotted', '1 cluster', '2 cluster', '3 cluster'))
if output_file:
with tb.open_file(output_file, mode="a") as out_file_h5:
cluster_array = np.array(n_cluster)
rec_array = np.array(zip(time_stamp, cluster_array[:, 0],
cluster_array[:, 1], cluster_array[:, 2],
cluster_array[:, 3], cluster_array[:, 4],
cluster_array[:, 5]),
dtype=[('time_stamp', float),
('cluster_0', float),
('cluster_1', float),
('cluster_2', float),
('cluster_3', float),
('cluster_4', float),
('cluster_5', float)
]).view(np.recarray)
try:
n_cluster_table = out_file_h5.create_table(
out_file_h5.root,
name='n_cluster',
description=rec_array,
title='Cluster per event',
filters=tb.Filters(complib='blosc',
complevel=5,
fletcher32=False))
n_cluster_table[:] = rec_array
except tb.exceptions.NodeError:
logging.warning(
output_file +
' has already a Beamspot note, do not overwrite existing.')
return time_stamp, n_cluster
def scan(self):
with PdfPages(self.output_filename + '.pdf') as output_pdf:
if self.test_tdc_values:
x, y, y_err = [], [], []
tdc_hist = None
self.fifo_readout.reset_sram_fifo() # clear fifo data
for pulse_width in [i for j in (range(10, 100, 5), range(100, 400, 10)) for i in j]:
logging.info('Test TDC for a pulse with of %d', pulse_width)
self.start_pulser(pulse_width, self.n_pulses)
time.sleep(self.n_pulses * pulse_width * 1e-9 + 0.1)
data = self.fifo_readout.read_data()
if data[is_tdc_word(data)].shape[0] != 0:
tdc_values = np.bitwise_and(data[is_tdc_word(data)], 0x00000FFF)
tdc_counter = np.bitwise_and(data[is_tdc_word(data)], 0x000FF000)
tdc_counter = np.right_shift(tdc_counter, 12)
if len(is_tdc_word(data)) != self.n_pulses:
logging.warning('%d TDC words instead of %d ', len(is_tdc_word(data)), self.n_pulses)
try:
if np.any(np.logical_and(tdc_counter[np.gradient(tdc_counter) != 1] != 0, tdc_counter[np.gradient(tdc_counter) != 1] != 255)):
logging.warning('The counter did not count correctly')
except ValueError:
logging.warning('The counter did not count correctly')
x.append(pulse_width)
y.append(np.mean(tdc_values))
y_err.append(np.std(tdc_values))
if tdc_hist is None:
tdc_hist = np.histogram(tdc_values, range=(0, 1023), bins=1024)[0]
else:
tdc_hist += np.histogram(tdc_values, range=(0, 1023), bins=1024)[0]
else:
logging.warning('No TDC words, check connection!')
plotting.plot_scatter(x, y, y_err, title='FPGA TDC linearity, ' + str(self.n_pulses) + ' each', x_label='Pulse width [ns]', y_label='TDC value', filename=output_pdf)
plotting.plot_scatter(x, y_err, title='FPGA TDC RMS, ' + str(self.n_pulses) + ' each', x_label='Pulse width [ns]', y_label='TDC RMS', filename=output_pdf)
if tdc_hist is not None:
plotting.plot_tdc_counter(tdc_hist, title='All TDC values', filename=output_pdf)
if self.test_trigger_delay:
x, y, y_err, y2, y2_err = [], [], [], [], []
self.fifo_readout.reset_sram_fifo() # clear fifo data
for pulse_delay in [i for j in (range(0, 100, 5), range(100, 500, 500)) for i in j]:
logging.info('Test TDC for a pulse delay of %d', pulse_delay)
for _ in range(10):
self.start_pulser(pulse_width=100, n_pulses=1, pulse_delay=pulse_delay)
time.sleep(0.1)
data = self.fifo_readout.read_data()
if data[is_tdc_word(data)].shape[0] != 0:
if len(is_tdc_word(data)) != 10:
logging.warning('%d TDC words instead of %d ', len(is_tdc_word(data)), 10)
tdc_values = np.bitwise_and(data[is_tdc_word(data)], 0x00000FFF)
tdc_delay = np.bitwise_and(data[is_tdc_word(data)], 0x0FF00000)
tdc_delay = np.right_shift(tdc_delay, 20)
x.append(pulse_delay)
y.append(np.mean(tdc_delay))
y_err.append(np.std(tdc_delay))
y2.append(np.mean(tdc_values))
y2_err.append(np.std(tdc_values))
else:
logging.warning('No TDC words, check connection!')
plotting.plot_scatter(x, y2, y2_err, title='FPGA TDC for different delays, ' + str(self.n_pulses) + ' each', x_label='Pulse delay [ns]', y_label='TDC value', filename=output_pdf)
plotting.plot_scatter(x, y, y_err, title='FPGA TDC trigger delay, ' + str(10) + ' each', x_label='Pulse delay [ns]', y_label='TDC trigger delay', filename=output_pdf)
plotting.plot_scatter(x, y_err, title='FPGA TDC trigger delay RMS, ' + str(10) + ' each', x_label='Pulse delay [ns]', y_label='TDC trigger delay RMS', filename=output_pdf)
def analyze_event_rate(scan_base,
combine_n_readouts=1000,
time_line_absolute=True,
output_pdf=None,
output_file=None):
''' Determines the number of events as a function of time. Therefore the data of a fixed number of read outs are combined ('combine_n_readouts'). The number of events is taken from the meta data info
and stored into a pdf file.
Parameters
----------
scan_base: list of str
scan base names (e.g.: ['//data//SCC_50_fei4_self_trigger_scan_390', ]
combine_n_readouts: int
the number of read outs to combine (e.g. 1000)
time_line_absolute: bool
if true the analysis uses absolute time stamps
output_pdf: PdfPages
PdfPages file object, if none the plot is printed to screen
'''
time_stamp = []
rate = []
start_time_set = False
for data_file in scan_base:
with tb.open_file(data_file + '_interpreted.h5',
mode="r") as in_file_h5:
meta_data_array = in_file_h5.root.meta_data[:]
parameter_ranges = np.column_stack(
(analysis_utils.get_ranges_from_array(
meta_data_array['timestamp_start'][::combine_n_readouts]),
analysis_utils.get_ranges_from_array(
meta_data_array['event_number'][::combine_n_readouts])))
if time_line_absolute:
time_stamp.extend(parameter_ranges[:-1, 0])
else:
if not start_time_set:
start_time = parameter_ranges[0, 0]
start_time_set = True
time_stamp.extend(
(parameter_ranges[:-1, 0] - start_time) / 60.0)
rate.extend((parameter_ranges[:-1, 3] - parameter_ranges[:-1, 2]) /
(parameter_ranges[:-1, 1] -
parameter_ranges[:-1, 0])) # d#Events / dt
if time_line_absolute:
plotting.plot_scatter_time(time_stamp,
rate,
title='Event rate [Hz]',
marker_style='o',
filename=output_pdf)
else:
plotting.plot_scatter(time_stamp,
rate,
title='Events per time',
x_label='Progressed time [min.]',
y_label='Events rate [Hz]',
marker_style='o',
filename=output_pdf)
if output_file:
with tb.open_file(output_file, mode="a") as out_file_h5:
rec_array = np.array(zip(time_stamp, rate),
dtype=[('time_stamp', float),
('rate', float)]).view(np.recarray)
try:
rate_table = out_file_h5.create_table(out_file_h5.root,
name='Eventrate',
description=rec_array,
title='Event rate',
filters=tb.Filters(
complib='blosc',
complevel=5,
fletcher32=False))
rate_table[:] = rec_array
except tb.exceptions.NodeError:
logging.warning(
output_file +
' has already a Eventrate note, do not overwrite existing.'
)
return time_stamp, rate
def analyze_beam_spot(scan_base,
combine_n_readouts=1000,
chunk_size=10000000,
plot_occupancy_hists=False,
output_pdf=None,
output_file=None):
''' Determines the mean x and y beam spot position as a function of time. Therefore the data of a fixed number of read outs are combined ('combine_n_readouts'). The occupancy is determined
for the given combined events and stored into a pdf file. At the end the beam x and y is plotted into a scatter plot with absolute positions in um.
Parameters
----------
scan_base: list of str
scan base names (e.g.: ['//data//SCC_50_fei4_self_trigger_scan_390', ]
combine_n_readouts: int
the number of read outs to combine (e.g. 1000)
max_chunk_size: int
the maximum chunk size used during read, if too big memory error occurs, if too small analysis takes longer
output_pdf: PdfPages
PdfPages file object, if none the plot is printed to screen
'''
time_stamp = []
x = []
y = []
for data_file in scan_base:
with tb.open_file(data_file + '_interpreted.h5',
mode="r+") as in_hit_file_h5:
# get data and data pointer
meta_data_array = in_hit_file_h5.root.meta_data[:]
hit_table = in_hit_file_h5.root.Hits
# determine the event ranges to analyze (timestamp_start, start_event_number, stop_event_number)
parameter_ranges = np.column_stack(
(analysis_utils.get_ranges_from_array(
meta_data_array['timestamp_start'][::combine_n_readouts]),
analysis_utils.get_ranges_from_array(
meta_data_array['event_number'][::combine_n_readouts])))
# create a event_numer index (important)
analysis_utils.index_event_number(hit_table)
# initialize the analysis and set settings
analyze_data = AnalyzeRawData()
analyze_data.create_tot_hist = False
analyze_data.create_bcid_hist = False
analyze_data.histogram.set_no_scan_parameter()
# variables for read speed up
index = 0 # index where to start the read out, 0 at the beginning, increased during looping
best_chunk_size = chunk_size
progress_bar = progressbar.ProgressBar(widgets=[
'',
progressbar.Percentage(), ' ',
progressbar.Bar(marker='*', left='|', right='|'), ' ',
progressbar.AdaptiveETA()
],
maxval=hit_table.shape[0],
term_width=80)
progress_bar.start()
# loop over the selected events
for parameter_index, parameter_range in enumerate(
parameter_ranges):
logging.debug('Analyze time stamp ' + str(parameter_range[0]) +
' and data from events = [' +
str(parameter_range[2]) + ',' +
str(parameter_range[3]) + '[ ' + str(
int(
float(
float(parameter_index) /
float(len(parameter_ranges)) *
100.0))) + '%')
analyze_data.reset() # resets the data of the last analysis
# loop over the hits in the actual selected events with optimizations: determine best chunk size, start word index given
readout_hit_len = 0 # variable to calculate a optimal chunk size value from the number of hits for speed up
for hits, index in analysis_utils.data_aligned_at_events(
hit_table,
start_event_number=parameter_range[2],
stop_event_number=parameter_range[3],
start_index=index,
chunk_size=best_chunk_size):
analyze_data.analyze_hits(
hits) # analyze the selected hits in chunks
readout_hit_len += hits.shape[0]
progress_bar.update(index)
best_chunk_size = int(1.5 * readout_hit_len) if int(
1.05 * readout_hit_len
) < chunk_size else chunk_size # to increase the readout speed, estimated the number of hits for one read instruction
# get and store results
occupancy_array = analyze_data.histogram.get_occupancy()
projection_x = np.sum(occupancy_array, axis=0).ravel()
projection_y = np.sum(occupancy_array, axis=1).ravel()
x.append(
analysis_utils.get_mean_from_histogram(projection_x,
bin_positions=range(
0, 80)))
y.append(
analysis_utils.get_mean_from_histogram(projection_y,
bin_positions=range(
0, 336)))
time_stamp.append(parameter_range[0])
if plot_occupancy_hists:
plotting.plot_occupancy(
occupancy_array[:, :, 0],
title='Occupancy for events between ' + time.strftime(
'%H:%M:%S', time.localtime(parameter_range[0])) +
' and ' + time.strftime(
'%H:%M:%S', time.localtime(parameter_range[1])),
filename=output_pdf)
progress_bar.finish()
plotting.plot_scatter([i * 250 for i in x], [i * 50 for i in y],
title='Mean beam position',
x_label='x [um]',
y_label='y [um]',
marker_style='-o',
filename=output_pdf)
if output_file:
with tb.open_file(output_file, mode="a") as out_file_h5:
rec_array = np.array(zip(time_stamp, x, y),
dtype=[('time_stamp', float), ('x', float),
('y', float)])
try:
beam_spot_table = out_file_h5.create_table(
out_file_h5.root,
name='Beamspot',
description=rec_array,
title='Beam spot position',
filters=tb.Filters(complib='blosc',
complevel=5,
fletcher32=False))
beam_spot_table[:] = rec_array
except tb.exceptions.NodeError:
logging.warning(
output_file +
' has already a Beamspot note, do not overwrite existing.')
return time_stamp, x, y
def analyze(self):
def analyze_raw_data_file(file_name):
with AnalyzeRawData(raw_data_file=file_name,
create_pdf=False) as analyze_raw_data:
analyze_raw_data.create_tot_hist = False
analyze_raw_data.create_fitted_threshold_hists = True
analyze_raw_data.create_threshold_mask = True
analyze_raw_data.interpreter.set_warning_output(
True
) # so far the data structure in a threshold scan was always bad, too many warnings given
analyze_raw_data.interpret_word_table()
def store_calibration_data_as_table(out_file_h5,
mean_threshold_calibration,
mean_threshold_rms_calibration,
threshold_calibration,
parameter_values):
logging.info("Storing calibration data in a table...")
filter_table = tb.Filters(complib='blosc',
complevel=5,
fletcher32=False)
mean_threshold_calib_table = out_file_h5.createTable(
out_file_h5.root,
name='MeanThresholdCalibration',
description=data_struct.MeanThresholdCalibrationTable,
title='mean_threshold_calibration',
filters=filter_table)
threshold_calib_table = out_file_h5.createTable(
out_file_h5.root,
name='ThresholdCalibration',
description=data_struct.ThresholdCalibrationTable,
title='threshold_calibration',
filters=filter_table)
for column in range(80):
for row in range(336):
for parameter_value_index, parameter_value in enumerate(
parameter_values):
threshold_calib_table.row['column'] = column
threshold_calib_table.row['row'] = row
threshold_calib_table.row[
'parameter_value'] = parameter_value
threshold_calib_table.row[
'threshold'] = threshold_calibration[
column, row, parameter_value_index]
threshold_calib_table.row.append()
for parameter_value_index, parameter_value in enumerate(
parameter_values):
mean_threshold_calib_table.row[
'parameter_value'] = parameter_value
mean_threshold_calib_table.row[
'mean_threshold'] = mean_threshold_calibration[
parameter_value_index]
mean_threshold_calib_table.row[
'threshold_rms'] = mean_threshold_rms_calibration[
parameter_value_index]
mean_threshold_calib_table.row.append()
threshold_calib_table.flush()
mean_threshold_calib_table.flush()
logging.info("done")
def store_calibration_data_as_array(out_file_h5,
mean_threshold_calibration,
mean_threshold_rms_calibration,
threshold_calibration):
logging.info("Storing calibration data in an array...")
filter_table = tb.Filters(complib='blosc',
complevel=5,
fletcher32=False)
mean_threshold_calib_array = out_file_h5.createCArray(
out_file_h5.root,
name='HistThresholdMeanCalibration',
atom=tb.Atom.from_dtype(mean_threshold_calibration.dtype),
shape=mean_threshold_calibration.shape,
title='mean_threshold_calibration',
filters=filter_table)
mean_threshold_calib_rms_array = out_file_h5.createCArray(
out_file_h5.root,
name='HistThresholdRMSCalibration',
atom=tb.Atom.from_dtype(mean_threshold_calibration.dtype),
shape=mean_threshold_calibration.shape,
title='mean_threshold_rms_calibration',
filters=filter_table)
threshold_calib_array = out_file_h5.createCArray(
out_file_h5.root,
name='HistThresholdCalibration',
atom=tb.Atom.from_dtype(threshold_calibration.dtype),
shape=threshold_calibration.shape,
title='threshold_calibration',
filters=filter_table)
mean_threshold_calib_array[:] = mean_threshold_calibration
mean_threshold_calib_rms_array[:] = mean_threshold_rms_calibration
threshold_calib_array[:] = threshold_calibration
logging.info("done")
def mask_columns(pixel_array, ignore_columns):
idx = np.array(ignore_columns) - 1 # from FE to Array columns
m = np.zeros_like(pixel_array)
m[:, idx] = 1
return np.ma.masked_array(pixel_array, m)
calibration_file = self.output_filename + '_calibration'
raw_data_files = analysis_utils.get_data_file_names_from_scan_base(
self.output_filename,
filter_file_words=['interpreted', 'calibration_calibration'])
parameter_name = self.scan_parameters._fields[1]
for raw_data_file in raw_data_files: # no using multithreading here, it is already used in fit
analyze_raw_data_file(raw_data_file)
files_per_parameter = analysis_utils.get_parameter_value_from_file_names(
[
file_name[:-3] + '_interpreted.h5'
for file_name in raw_data_files
], parameter_name)
logging.info("Create calibration from data")
with tb.openFile(
self.output_filename + '.h5',
mode="r") as in_file_h5: # deduce settings from raw data file
ignore_columns = in_file_h5.root.configuration.run_conf[:][
np.where(in_file_h5.root.configuration.run_conf[:]['name'] ==
'ignore_columns')]['value'][0]
ignore_columns = ast.literal_eval(ignore_columns)
mean_threshold_calibration = np.empty(shape=(len(raw_data_files), ),
dtype='<f8')
mean_threshold_rms_calibration = np.empty(
shape=(len(raw_data_files), ), dtype='<f8')
threshold_calibration = np.empty(shape=(80, 336, len(raw_data_files)),
dtype='<f8')
if self.create_plots:
logging.info('Saving calibration plots in: %s' %
(calibration_file + '.pdf'))
output_pdf = PdfPages(calibration_file + '.pdf')
parameter_values = []
for index, (analyzed_data_file,
parameters) in enumerate(files_per_parameter.items()):
parameter_values.append(parameters.values()[0][0])
with tb.openFile(analyzed_data_file, mode="r") as in_file_h5:
occupancy_masked = mask_columns(
pixel_array=in_file_h5.root.HistOcc[:],
ignore_columns=ignore_columns
) # mask the not scanned columns for analysis and plotting
thresholds_masked = mask_columns(
pixel_array=in_file_h5.root.HistThresholdFitted[:],
ignore_columns=ignore_columns)
if self.create_plots:
plotThreeWay(hist=thresholds_masked,
title='Threshold Fitted for ' +
parameters.keys()[0] + ' = ' +
str(parameters.values()[0][0]),
filename=output_pdf)
plsr_dacs = analysis_utils.get_scan_parameter(
meta_data_array=in_file_h5.root.meta_data[:]
)['PlsrDAC']
plot_scurves(occupancy_hist=occupancy_masked,
scan_parameters=plsr_dacs,
scan_parameter_name='PlsrDAC',
filename=output_pdf)
# fill the calibration data arrays
mean_threshold_calibration[index] = np.ma.mean(
thresholds_masked)
mean_threshold_rms_calibration[index] = np.ma.std(
thresholds_masked)
threshold_calibration[:, :, index] = thresholds_masked.T
with tb.openFile(calibration_file + '.h5', mode="w") as out_file_h5:
store_calibration_data_as_array(
out_file_h5=out_file_h5,
mean_threshold_calibration=mean_threshold_calibration,
mean_threshold_rms_calibration=mean_threshold_rms_calibration,
threshold_calibration=threshold_calibration)
store_calibration_data_as_table(
out_file_h5=out_file_h5,
mean_threshold_calibration=mean_threshold_calibration,
mean_threshold_rms_calibration=mean_threshold_rms_calibration,
threshold_calibration=threshold_calibration,
parameter_values=parameter_values)
if self.create_plots:
plot_scatter(x=parameter_values,
y=mean_threshold_calibration,
title='Threshold calibration',
x_label=parameter_name,
y_label='Mean threshold',
log_x=False,
filename=output_pdf)
plot_scatter(x=parameter_values,
y=mean_threshold_calibration,
title='Threshold calibration',
x_label=parameter_name,
y_label='Mean threshold',
log_x=True,
filename=output_pdf)
output_pdf.close()
def analyze_injected_charge(data_analyzed_file):
logging.info('Analyze the injected charge')
with tb.openFile(data_analyzed_file, mode="r") as in_file_h5:
occupancy = in_file_h5.root.HistOcc[:]
gdacs = analysis_utils.get_scan_parameter(
in_file_h5.root.meta_data[:])['GDAC']
with tb.openFile(
analysis_configuration['input_file_calibration'], mode="r"
) as in_file_calibration_h5: # read calibration file from calibrate_threshold_gdac scan
mean_threshold_calibration = in_file_calibration_h5.root.MeanThresholdCalibration[:]
threshold_calibration_array = in_file_calibration_h5.root.HistThresholdCalibration[:]
gdac_range_calibration = mean_threshold_calibration['gdac']
gdac_range_source_scan = gdacs
logging.info(
'Analyzing source scan data with %d GDAC settings from %d to %d with minimum step sizes from %d to %d'
% (len(gdac_range_source_scan), np.min(gdac_range_source_scan),
np.max(gdac_range_source_scan),
np.min(np.gradient(gdac_range_source_scan)),
np.max(np.gradient(gdac_range_source_scan))))
logging.info(
'Use calibration data with %d GDAC settings from %d to %d with minimum step sizes from %d to %d'
% (len(gdac_range_calibration), np.min(gdac_range_calibration),
np.max(gdac_range_calibration),
np.min(np.gradient(gdac_range_calibration)),
np.max(np.gradient(gdac_range_calibration))))
# rate_normalization of the total hit number for each GDAC setting
rate_normalization = 1.
if analysis_configuration['normalize_rate']:
rate_normalization = analysis_utils.get_rate_normalization(
hit_file=hit_file,
cluster_file=hit_file,
parameter='GDAC',
reference=analysis_configuration[
'normalization_reference'],
plot=analysis_configuration['plot_normalization'])
# correcting the hit numbers for the different cluster sizes
correction_factors = 1.
if analysis_configuration['use_cluster_rate_correction']:
correction_h5 = tb.openFile(cluster_sizes_file, mode="r")
cluster_size_histogram = correction_h5.root.AllHistClusterSize[:]
correction_factors = analysis_utils.get_hit_rate_correction(
gdacs=gdac_range_source_scan,
calibration_gdacs=gdac_range_source_scan,
cluster_size_histogram=cluster_size_histogram)
if analysis_configuration['plot_cluster_sizes']:
plot_cluster_sizes(correction_h5,
in_file_calibration_h5,
gdac_range=gdac_range_source_scan)
print correction_factors
pixel_thresholds = analysis_utils.get_pixel_thresholds_from_calibration_array(
gdacs=gdac_range_source_scan,
calibration_gdacs=gdac_range_calibration,
threshold_calibration_array=threshold_calibration_array
) # interpolates the threshold at the source scan GDAC setting from the calibration
pixel_hits = np.swapaxes(
occupancy, 0, 1) # create hit array with shape (col, row, ...)
pixel_hits = pixel_hits * correction_factors * rate_normalization
# choose region with pixels that have a sufficient occupancy but are not too hot
good_pixel = analysis_utils.select_good_pixel_region(
pixel_hits,
col_span=analysis_configuration['col_span'],
row_span=analysis_configuration['row_span'],
min_cut_threshold=analysis_configuration['min_cut_threshold'],
max_cut_threshold=analysis_configuration['max_cut_threshold'])
pixel_mask = ~np.ma.getmaskarray(good_pixel)
selected_pixel_hits = pixel_hits[
pixel_mask, :] # reduce the data to pixels that are in the good pixel region
selected_pixel_thresholds = pixel_thresholds[
pixel_mask, :] # reduce the data to pixels that are in the good pixel region
plotting.plot_occupancy(good_pixel.T,
title='Select ' +
str(len(selected_pixel_hits)) +
' pixels for analysis')
# reshape to one dimension
x = selected_pixel_thresholds.flatten()
y = selected_pixel_hits.flatten()
#nothing should be NAN, NAN is not supported yet
if np.isfinite(x).shape != x.shape or np.isfinite(
y).shape != y.shape:
logging.warning(
'There are pixels with NaN or INF threshold or hit values, analysis will fail'
)
# calculated profile histogram
x_p, y_p, y_p_e = analysis_utils.get_profile_histogram(
x, y, n_bins=analysis_configuration['n_bins']
) # profile histogram data
# select only the data point where the calibration worked
selected_data = np.logical_and(
x_p > analysis_configuration['min_thr'] /
analysis_configuration['vcal_calibration'],
x_p < analysis_configuration['max_thr'] /
analysis_configuration['vcal_calibration'])
x_p = x_p[selected_data]
y_p = y_p[selected_data]
y_p_e = y_p_e[selected_data]
plot_result(x_p, y_p, y_p_e)
# calculate and plot mean results
x_mean = analysis_utils.get_mean_threshold_from_calibration(
gdac_range_source_scan, mean_threshold_calibration)
y_mean = selected_pixel_hits.mean(axis=(0))
plotting.plot_scatter(
np.array(gdac_range_source_scan),
y_mean,
log_x=True,
plot_range=None,
title='Mean single pixel cluster rate at different thresholds',
x_label='threshold setting [GDAC]',
y_label='mean single pixel cluster rate')
plotting.plot_scatter(
x_mean * analysis_configuration['vcal_calibration'],
y_mean,
plot_range=(analysis_configuration['min_thr'],
analysis_configuration['max_thr']),
title='Mean single pixel cluster rate at different thresholds',
x_label='mean threshold [e]',
y_label='mean single pixel cluster rate')
if analysis_configuration['use_cluster_rate_correction']:
correction_h5.close()
def analyse_n_cluster_per_event(
scan_base,
include_no_cluster=False,
time_line_absolute=True,
combine_n_readouts=1000,
chunk_size=10000000,
plot_n_cluster_hists=False,
output_pdf=None,
output_file=None,
):
""" Determines the number of cluster per event as a function of time. Therefore the data of a fixed number of read outs are combined ('combine_n_readouts').
Parameters
----------
scan_base: list of str
scan base names (e.g.: ['//data//SCC_50_fei4_self_trigger_scan_390', ]
include_no_cluster: bool
Set to true to also consider all events without any hit.
combine_n_readouts: int
the number of read outs to combine (e.g. 1000)
max_chunk_size: int
the maximum chunk size used during read, if too big memory error occurs, if too small analysis takes longer
output_pdf: PdfPages
PdfPages file object, if none the plot is printed to screen
"""
time_stamp = []
n_cluster = []
start_time_set = False
for data_file in scan_base:
with tb.openFile(data_file + "_interpreted.h5", mode="r+") as in_cluster_file_h5:
# get data and data pointer
meta_data_array = in_cluster_file_h5.root.meta_data[:]
cluster_table = in_cluster_file_h5.root.Cluster
# determine the event ranges to analyze (timestamp_start, start_event_number, stop_event_number)
parameter_ranges = np.column_stack(
(
analysis_utils.get_ranges_from_array(meta_data_array["timestamp_start"][::combine_n_readouts]),
analysis_utils.get_ranges_from_array(meta_data_array["event_number"][::combine_n_readouts]),
)
)
# create a event_numer index (important for speed)
analysis_utils.index_event_number(cluster_table)
# initialize the analysis and set settings
analyze_data = AnalyzeRawData()
analyze_data.create_tot_hist = False
analyze_data.create_bcid_hist = False
# variables for read speed up
index = 0 # index where to start the read out, 0 at the beginning, increased during looping
best_chunk_size = chunk_size
total_cluster = cluster_table.shape[0]
progress_bar = progressbar.ProgressBar(
widgets=[
"",
progressbar.Percentage(),
" ",
progressbar.Bar(marker="*", left="|", right="|"),
" ",
analysis_utils.ETA(),
],
maxval=total_cluster,
term_width=80,
)
progress_bar.start()
# loop over the selected events
for parameter_index, parameter_range in enumerate(parameter_ranges):
logging.debug(
"Analyze time stamp "
+ str(parameter_range[0])
+ " and data from events = ["
+ str(parameter_range[2])
+ ","
+ str(parameter_range[3])
+ "[ "
+ str(int(float(float(parameter_index) / float(len(parameter_ranges)) * 100.0)))
+ "%"
)
analyze_data.reset() # resets the data of the last analysis
# loop over the cluster in the actual selected events with optimizations: determine best chunk size, start word index given
readout_cluster_len = (
0
) # variable to calculate a optimal chunk size value from the number of hits for speed up
hist = None
for clusters, index in analysis_utils.data_aligned_at_events(
cluster_table,
start_event_number=parameter_range[2],
stop_event_number=parameter_range[3],
start=index,
chunk_size=best_chunk_size,
):
n_cluster_per_event = analysis_utils.get_n_cluster_in_events(clusters["event_number"])[
:, 1
] # array with the number of cluster per event, cluster per event are at least 1
if hist is None:
hist = np.histogram(n_cluster_per_event, bins=10, range=(0, 10))[0]
else:
hist = np.add(hist, np.histogram(n_cluster_per_event, bins=10, range=(0, 10))[0])
if include_no_cluster and parameter_range[3] is not None: # happend for the last readout
hist[0] = (parameter_range[3] - parameter_range[2]) - len(
n_cluster_per_event
) # add the events without any cluster
readout_cluster_len += clusters.shape[0]
total_cluster -= len(clusters)
progress_bar.update(index)
best_chunk_size = (
int(1.5 * readout_cluster_len) if int(1.05 * readout_cluster_len) < chunk_size else chunk_size
) # to increase the readout speed, estimated the number of hits for one read instruction
if plot_n_cluster_hists:
plotting.plot_1d_hist(
hist,
title="Number of cluster per event at " + str(parameter_range[0]),
x_axis_title="Number of cluster",
y_axis_title="#",
log_y=True,
filename=output_pdf,
)
hist = hist.astype("f4") / np.sum(hist) # calculate fraction from total numbers
if time_line_absolute:
time_stamp.append(parameter_range[0])
else:
if not start_time_set:
start_time = parameter_ranges[0, 0]
start_time_set = True
time_stamp.append((parameter_range[0] - start_time) / 60.0)
n_cluster.append(hist)
progress_bar.finish()
if total_cluster != 0:
logging.warning("Not all clusters were selected during analysis. Analysis is therefore not exact")
if time_line_absolute:
plotting.plot_scatter_time(
time_stamp,
n_cluster,
title="Number of cluster per event as a function of time",
marker_style="o",
filename=output_pdf,
legend=("0 cluster", "1 cluster", "2 cluster", "3 cluster")
if include_no_cluster
else ("0 cluster not plotted", "1 cluster", "2 cluster", "3 cluster"),
)
else:
plotting.plot_scatter(
time_stamp,
n_cluster,
title="Number of cluster per event as a function of time",
x_label="time [min.]",
marker_style="o",
filename=output_pdf,
legend=("0 cluster", "1 cluster", "2 cluster", "3 cluster")
if include_no_cluster
else ("0 cluster not plotted", "1 cluster", "2 cluster", "3 cluster"),
)
if output_file:
with tb.openFile(output_file, mode="a") as out_file_h5:
cluster_array = np.array(n_cluster)
rec_array = np.array(
zip(
time_stamp,
cluster_array[:, 0],
cluster_array[:, 1],
cluster_array[:, 2],
cluster_array[:, 3],
cluster_array[:, 4],
cluster_array[:, 5],
),
dtype=[
("time_stamp", float),
("cluster_0", float),
("cluster_1", float),
("cluster_2", float),
("cluster_3", float),
("cluster_4", float),
("cluster_5", float),
],
).view(np.recarray)
try:
n_cluster_table = out_file_h5.createTable(
out_file_h5.root,
name="n_cluster",
description=rec_array,
title="Cluster per event",
filters=tb.Filters(complib="blosc", complevel=5, fletcher32=False),
)
n_cluster_table[:] = rec_array
except tb.exceptions.NodeError:
logging.warning(output_file + " has already a Beamspot note, do not overwrite existing.")
return time_stamp, n_cluster
def analyze_beam_spot(
scan_base,
combine_n_readouts=1000,
chunk_size=10000000,
plot_occupancy_hists=False,
output_pdf=None,
output_file=None,
):
""" Determines the mean x and y beam spot position as a function of time. Therefore the data of a fixed number of read outs are combined ('combine_n_readouts'). The occupancy is determined
for the given combined events and stored into a pdf file. At the end the beam x and y is plotted into a scatter plot with absolute positions in um.
Parameters
----------
scan_base: list of str
scan base names (e.g.: ['//data//SCC_50_fei4_self_trigger_scan_390', ]
combine_n_readouts: int
the number of read outs to combine (e.g. 1000)
max_chunk_size: int
the maximum chunk size used during read, if too big memory error occurs, if too small analysis takes longer
output_pdf: PdfPages
PdfPages file object, if none the plot is printed to screen
"""
time_stamp = []
x = []
y = []
for data_file in scan_base:
with tb.openFile(data_file + "_interpreted.h5", mode="r+") as in_hit_file_h5:
# get data and data pointer
meta_data_array = in_hit_file_h5.root.meta_data[:]
hit_table = in_hit_file_h5.root.Hits
# determine the event ranges to analyze (timestamp_start, start_event_number, stop_event_number)
parameter_ranges = np.column_stack(
(
analysis_utils.get_ranges_from_array(meta_data_array["timestamp_start"][::combine_n_readouts]),
analysis_utils.get_ranges_from_array(meta_data_array["event_number"][::combine_n_readouts]),
)
)
# create a event_numer index (important)
analysis_utils.index_event_number(hit_table)
# initialize the analysis and set settings
analyze_data = AnalyzeRawData()
analyze_data.create_tot_hist = False
analyze_data.create_bcid_hist = False
analyze_data.histograming.set_no_scan_parameter()
# variables for read speed up
index = 0 # index where to start the read out, 0 at the beginning, increased during looping
best_chunk_size = chunk_size
progress_bar = progressbar.ProgressBar(
widgets=[
"",
progressbar.Percentage(),
" ",
progressbar.Bar(marker="*", left="|", right="|"),
" ",
analysis_utils.ETA(),
],
maxval=hit_table.shape[0],
term_width=80,
)
progress_bar.start()
# loop over the selected events
for parameter_index, parameter_range in enumerate(parameter_ranges):
logging.debug(
"Analyze time stamp "
+ str(parameter_range[0])
+ " and data from events = ["
+ str(parameter_range[2])
+ ","
+ str(parameter_range[3])
+ "[ "
+ str(int(float(float(parameter_index) / float(len(parameter_ranges)) * 100.0)))
+ "%"
)
analyze_data.reset() # resets the data of the last analysis
# loop over the hits in the actual selected events with optimizations: determine best chunk size, start word index given
readout_hit_len = (
0
) # variable to calculate a optimal chunk size value from the number of hits for speed up
for hits, index in analysis_utils.data_aligned_at_events(
hit_table,
start_event_number=parameter_range[2],
stop_event_number=parameter_range[3],
start=index,
chunk_size=best_chunk_size,
):
analyze_data.analyze_hits(hits) # analyze the selected hits in chunks
readout_hit_len += hits.shape[0]
progress_bar.update(index)
best_chunk_size = (
int(1.5 * readout_hit_len) if int(1.05 * readout_hit_len) < chunk_size else chunk_size
) # to increase the readout speed, estimated the number of hits for one read instruction
# get and store results
occupancy_array = analyze_data.histograming.get_occupancy()
projection_x = np.sum(occupancy_array, axis=0).ravel()
projection_y = np.sum(occupancy_array, axis=1).ravel()
x.append(analysis_utils.get_mean_from_histogram(projection_x, bin_positions=range(0, 80)))
y.append(analysis_utils.get_mean_from_histogram(projection_y, bin_positions=range(0, 336)))
time_stamp.append(parameter_range[0])
if plot_occupancy_hists:
plotting.plot_occupancy(
occupancy_array[:, :, 0],
title="Occupancy for events between "
+ time.strftime("%H:%M:%S", time.localtime(parameter_range[0]))
+ " and "
+ time.strftime("%H:%M:%S", time.localtime(parameter_range[1])),
filename=output_pdf,
)
progress_bar.finish()
plotting.plot_scatter(
[i * 250 for i in x],
[i * 50 for i in y],
title="Mean beam position",
x_label="x [um]",
y_label="y [um]",
marker_style="-o",
filename=output_pdf,
)
if output_file:
with tb.openFile(output_file, mode="a") as out_file_h5:
rec_array = np.array(zip(time_stamp, x, y), dtype=[("time_stamp", float), ("x", float), ("y", float)])
try:
beam_spot_table = out_file_h5.createTable(
out_file_h5.root,
name="Beamspot",
description=rec_array,
title="Beam spot position",
filters=tb.Filters(complib="blosc", complevel=5, fletcher32=False),
)
beam_spot_table[:] = rec_array
except tb.exceptions.NodeError:
logging.warning(output_file + " has already a Beamspot note, do not overwrite existing.")
return time_stamp, x, y
def analyze_event_rate(scan_base, combine_n_readouts=1000, time_line_absolute=True, output_pdf=None, output_file=None):
""" Determines the number of events as a function of time. Therefore the data of a fixed number of read outs are combined ('combine_n_readouts'). The number of events is taken from the meta data info
and stored into a pdf file.
Parameters
----------
scan_base: list of str
scan base names (e.g.: ['//data//SCC_50_fei4_self_trigger_scan_390', ]
combine_n_readouts: int
the number of read outs to combine (e.g. 1000)
time_line_absolute: bool
if true the analysis uses absolute time stamps
output_pdf: PdfPages
PdfPages file object, if none the plot is printed to screen
"""
time_stamp = []
rate = []
start_time_set = False
for data_file in scan_base:
with tb.openFile(data_file + "_interpreted.h5", mode="r") as in_file_h5:
meta_data_array = in_file_h5.root.meta_data[:]
parameter_ranges = np.column_stack(
(
analysis_utils.get_ranges_from_array(meta_data_array["timestamp_start"][::combine_n_readouts]),
analysis_utils.get_ranges_from_array(meta_data_array["event_number"][::combine_n_readouts]),
)
)
if time_line_absolute:
time_stamp.extend(parameter_ranges[:-1, 0])
else:
if not start_time_set:
start_time = parameter_ranges[0, 0]
start_time_set = True
time_stamp.extend((parameter_ranges[:-1, 0] - start_time) / 60.0)
rate.extend(
(parameter_ranges[:-1, 3] - parameter_ranges[:-1, 2])
/ (parameter_ranges[:-1, 1] - parameter_ranges[:-1, 0])
) # d#Events / dt
if time_line_absolute:
plotting.plot_scatter_time(time_stamp, rate, title="Event rate [Hz]", marker_style="o", filename=output_pdf)
else:
plotting.plot_scatter(
time_stamp,
rate,
title="Events per time",
x_label="Progressed time [min.]",
y_label="Events rate [Hz]",
marker_style="o",
filename=output_pdf,
)
if output_file:
with tb.openFile(output_file, mode="a") as out_file_h5:
rec_array = np.array(zip(time_stamp, rate), dtype=[("time_stamp", float), ("rate", float)]).view(
np.recarray
)
try:
rate_table = out_file_h5.createTable(
out_file_h5.root,
name="Eventrate",
description=rec_array,
title="Event rate",
filters=tb.Filters(complib="blosc", complevel=5, fletcher32=False),
)
rate_table[:] = rec_array
except tb.exceptions.NodeError:
logging.warning(output_file + " has already a Eventrate note, do not overwrite existing.")
return time_stamp, rate
def scan(self):
with PdfPages(self.output_filename + ".pdf") as output_pdf:
if self.test_tdc_values:
x, y, y_err = [], [], []
tdc_hist = None
self.fifo_readout.reset_sram_fifo() # clear fifo data
for pulse_width in [i for j in (range(10, 100, 5), range(100, 400, 10)) for i in j]:
logging.info("Test TDC for a pulse with of %d", pulse_width)
self.start_pulser(pulse_width, self.n_pulses)
time.sleep(self.n_pulses * pulse_width * 1e-9 + 0.1)
data = self.fifo_readout.read_data()
if data[is_tdc_word(data)].shape[0] != 0:
tdc_values = np.bitwise_and(data[is_tdc_word(data)], 0x00000FFF)
tdc_counter = np.bitwise_and(data[is_tdc_word(data)], 0x000FF000)
tdc_counter = np.right_shift(tdc_counter, 12)
if len(is_tdc_word(data)) != self.n_pulses:
logging.warning("%d TDC words instead of %d ", len(is_tdc_word(data)), self.n_pulses)
try:
if np.any(
np.logical_and(
tdc_counter[np.gradient(tdc_counter) != 1] != 0,
tdc_counter[np.gradient(tdc_counter) != 1] != 255,
)
):
logging.warning("The counter did not count correctly")
except ValueError:
logging.warning("The counter did not count correctly")
x.append(pulse_width)
y.append(np.mean(tdc_values))
y_err.append(np.std(tdc_values))
if tdc_hist is None:
tdc_hist = np.histogram(tdc_values, range=(0, 1023), bins=1024)[0]
else:
tdc_hist += np.histogram(tdc_values, range=(0, 1023), bins=1024)[0]
else:
logging.warning("No TDC words, check connection!")
plotting.plot_scatter(
x,
y,
y_err,
title="FPGA TDC linearity, " + str(self.n_pulses) + " each",
x_label="Pulse width [ns]",
y_label="TDC value",
filename=output_pdf,
)
plotting.plot_scatter(
x,
y_err,
title="FPGA TDC RMS, " + str(self.n_pulses) + " each",
x_label="Pulse width [ns]",
y_label="TDC RMS",
filename=output_pdf,
)
if tdc_hist is not None:
plotting.plot_tdc_counter(tdc_hist, title="All TDC values", filename=output_pdf)
if self.test_trigger_delay:
x, y, y_err, y2, y2_err = [], [], [], [], []
self.fifo_readout.reset_sram_fifo() # clear fifo data
for pulse_delay in [i for j in (range(0, 100, 5), range(100, 500, 500)) for i in j]:
logging.info("Test TDC for a pulse delay of %d", pulse_delay)
for _ in range(10):
self.start_pulser(pulse_width=100, n_pulses=1, pulse_delay=pulse_delay)
time.sleep(0.1)
data = self.fifo_readout.read_data()
if data[is_tdc_word(data)].shape[0] != 0:
if len(is_tdc_word(data)) != 10:
logging.warning("%d TDC words instead of %d ", len(is_tdc_word(data)), 10)
tdc_values = np.bitwise_and(data[is_tdc_word(data)], 0x00000FFF)
tdc_delay = np.bitwise_and(data[is_tdc_word(data)], 0x0FF00000)
tdc_delay = np.right_shift(tdc_delay, 20)
x.append(pulse_delay)
y.append(np.mean(tdc_delay))
y_err.append(np.std(tdc_delay))
y2.append(np.mean(tdc_values))
y2_err.append(np.std(tdc_values))
else:
logging.warning("No TDC words, check connection!")
plotting.plot_scatter(
x,
y2,
y2_err,
title="FPGA TDC for different delays, " + str(self.n_pulses) + " each",
x_label="Pulse delay [ns]",
y_label="TDC value",
filename=output_pdf,
)
plotting.plot_scatter(
x,
y,
y_err,
title="FPGA TDC trigger delay, " + str(10) + " each",
x_label="Pulse delay [ns]",
y_label="TDC trigger delay",
filename=output_pdf,
)
plotting.plot_scatter(
x,
y_err,
title="FPGA TDC trigger delay RMS, " + str(10) + " each",
x_label="Pulse delay [ns]",
y_label="TDC trigger delay RMS",
filename=output_pdf,
)
def create_threshold_calibration(scan_base_file_name, create_plots=True): # Create calibration function, can be called stand alone
def analyze_raw_data_file(file_name):
if os.path.isfile(file_name[:-3] + '_interpreted.h5'): # skip analysis if already done
logging.warning('Analyzed data file ' + file_name + ' already exists. Skip analysis for this file.')
else:
with AnalyzeRawData(raw_data_file=file_name, create_pdf=False) as analyze_raw_data:
analyze_raw_data.create_tot_hist = False
analyze_raw_data.create_tot_pixel_hist = False
analyze_raw_data.create_fitted_threshold_hists = True
analyze_raw_data.create_threshold_mask = True
analyze_raw_data.interpreter.set_warning_output(False) # RX errors would fill the console
analyze_raw_data.interpret_word_table()
def store_calibration_data_as_table(out_file_h5, mean_threshold_calibration, mean_threshold_rms_calibration, threshold_calibration, parameter_values):
logging.info("Storing calibration data in a table...")
filter_table = tb.Filters(complib='blosc', complevel=5, fletcher32=False)
mean_threshold_calib_table = out_file_h5.createTable(out_file_h5.root, name='MeanThresholdCalibration', description=data_struct.MeanThresholdCalibrationTable, title='mean_threshold_calibration', filters=filter_table)
threshold_calib_table = out_file_h5.createTable(out_file_h5.root, name='ThresholdCalibration', description=data_struct.ThresholdCalibrationTable, title='threshold_calibration', filters=filter_table)
for column in range(80):
for row in range(336):
for parameter_value_index, parameter_value in enumerate(parameter_values):
threshold_calib_table.row['column'] = column
threshold_calib_table.row['row'] = row
threshold_calib_table.row['parameter_value'] = parameter_value
threshold_calib_table.row['threshold'] = threshold_calibration[column, row, parameter_value_index]
threshold_calib_table.row.append()
for parameter_value_index, parameter_value in enumerate(parameter_values):
mean_threshold_calib_table.row['parameter_value'] = parameter_value
mean_threshold_calib_table.row['mean_threshold'] = mean_threshold_calibration[parameter_value_index]
mean_threshold_calib_table.row['threshold_rms'] = mean_threshold_rms_calibration[parameter_value_index]
mean_threshold_calib_table.row.append()
threshold_calib_table.flush()
mean_threshold_calib_table.flush()
logging.info("done")
def store_calibration_data_as_array(out_file_h5, mean_threshold_calibration, mean_threshold_rms_calibration, threshold_calibration, parameter_name, parameter_values):
logging.info("Storing calibration data in an array...")
filter_table = tb.Filters(complib='blosc', complevel=5, fletcher32=False)
mean_threshold_calib_array = out_file_h5.createCArray(out_file_h5.root, name='HistThresholdMeanCalibration', atom=tb.Atom.from_dtype(mean_threshold_calibration.dtype), shape=mean_threshold_calibration.shape, title='mean_threshold_calibration', filters=filter_table)
mean_threshold_calib_rms_array = out_file_h5.createCArray(out_file_h5.root, name='HistThresholdRMSCalibration', atom=tb.Atom.from_dtype(mean_threshold_calibration.dtype), shape=mean_threshold_calibration.shape, title='mean_threshold_rms_calibration', filters=filter_table)
threshold_calib_array = out_file_h5.createCArray(out_file_h5.root, name='HistThresholdCalibration', atom=tb.Atom.from_dtype(threshold_calibration.dtype), shape=threshold_calibration.shape, title='threshold_calibration', filters=filter_table)
mean_threshold_calib_array[:] = mean_threshold_calibration
mean_threshold_calib_rms_array[:] = mean_threshold_rms_calibration
threshold_calib_array[:] = threshold_calibration
mean_threshold_calib_array.attrs.dimensions = ['column', 'row', parameter_name]
mean_threshold_calib_rms_array.attrs.dimensions = ['column', 'row', parameter_name]
threshold_calib_array.attrs.dimensions = ['column', 'row', parameter_name]
mean_threshold_calib_array.attrs.scan_parameter_values = parameter_values
mean_threshold_calib_rms_array.attrs.scan_parameter_values = parameter_values
threshold_calib_array.attrs.scan_parameter_values = parameter_values
logging.info("done")
def mask_columns(pixel_array, ignore_columns):
idx = np.array(ignore_columns) - 1 # from FE to Array columns
m = np.zeros_like(pixel_array)
m[:, idx] = 1
return np.ma.masked_array(pixel_array, m)
raw_data_files = analysis_utils.get_data_file_names_from_scan_base(scan_base_file_name, filter_file_words=['interpreted', 'calibration_calibration'])
first_scan_base_file_name = scan_base_file_name if isinstance(scan_base_file_name, basestring) else scan_base_file_name[0] # multilpe scan_base_file_names for multiple runs
with tb.openFile(first_scan_base_file_name + '.h5', mode="r") as in_file_h5: # deduce scan parameters from the first (and often only) scan base file name
ignore_columns = in_file_h5.root.configuration.run_conf[:][np.where(in_file_h5.root.configuration.run_conf[:]['name'] == 'ignore_columns')]['value'][0]
parameter_name = in_file_h5.root.configuration.run_conf[:][np.where(in_file_h5.root.configuration.run_conf[:]['name'] == 'scan_parameters')]['value'][0]
ignore_columns = ast.literal_eval(ignore_columns)
parameter_name = ast.literal_eval(parameter_name)[1][0]
calibration_file = first_scan_base_file_name + '_calibration'
for raw_data_file in raw_data_files: # analyze each raw data file, not using multithreading here, it is already used in s-curve fit
analyze_raw_data_file(raw_data_file)
files_per_parameter = analysis_utils.get_parameter_value_from_file_names([file_name[:-3] + '_interpreted.h5' for file_name in raw_data_files], parameter_name, unique=True, sort=True)
logging.info("Create calibration from data")
mean_threshold_calibration = np.empty(shape=(len(raw_data_files),), dtype='<f8')
mean_threshold_rms_calibration = np.empty(shape=(len(raw_data_files),), dtype='<f8')
threshold_calibration = np.empty(shape=(80, 336, len(raw_data_files)), dtype='<f8')
if create_plots:
logging.info('Saving calibration plots in: %s', calibration_file + '.pdf')
output_pdf = PdfPages(calibration_file + '.pdf')
progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=len(files_per_parameter.items()), term_width=80)
progress_bar.start()
parameter_values = []
for index, (analyzed_data_file, parameters) in enumerate(files_per_parameter.items()):
parameter_values.append(parameters.values()[0][0])
with tb.openFile(analyzed_data_file, mode="r") as in_file_h5:
occupancy_masked = mask_columns(pixel_array=in_file_h5.root.HistOcc[:], ignore_columns=ignore_columns) # mask the not scanned columns for analysis and plotting
thresholds_masked = mask_columns(pixel_array=in_file_h5.root.HistThresholdFitted[:], ignore_columns=ignore_columns)
if create_plots:
plot_three_way(hist=thresholds_masked, title='Threshold Fitted for ' + parameters.keys()[0] + ' = ' + str(parameters.values()[0][0]), filename=output_pdf)
plsr_dacs = analysis_utils.get_scan_parameter(meta_data_array=in_file_h5.root.meta_data[:])['PlsrDAC']
plot_scurves(occupancy_hist=occupancy_masked, scan_parameters=plsr_dacs, scan_parameter_name='PlsrDAC', filename=output_pdf)
# fill the calibration data arrays
mean_threshold_calibration[index] = np.ma.mean(thresholds_masked)
mean_threshold_rms_calibration[index] = np.ma.std(thresholds_masked)
threshold_calibration[:, :, index] = thresholds_masked.T
progress_bar.update(index)
progress_bar.finish()
with tb.openFile(calibration_file + '.h5', mode="w") as out_file_h5:
store_calibration_data_as_array(out_file_h5=out_file_h5, mean_threshold_calibration=mean_threshold_calibration, mean_threshold_rms_calibration=mean_threshold_rms_calibration, threshold_calibration=threshold_calibration, parameter_name=parameter_name, parameter_values=parameter_values)
store_calibration_data_as_table(out_file_h5=out_file_h5, mean_threshold_calibration=mean_threshold_calibration, mean_threshold_rms_calibration=mean_threshold_rms_calibration, threshold_calibration=threshold_calibration, parameter_values=parameter_values)
if create_plots:
plot_scatter(x=parameter_values, y=mean_threshold_calibration, title='Threshold calibration', x_label=parameter_name, y_label='Mean threshold', log_x=False, filename=output_pdf)
plot_scatter(x=parameter_values, y=mean_threshold_calibration, title='Threshold calibration', x_label=parameter_name, y_label='Mean threshold', log_x=True, filename=output_pdf)
output_pdf.close()