def main(
input_file,
output_file,
max_events=None,
pixel_id=[...],
integral_width=7,
shift=0,
pulse_finder_threshold=3.,
use_digicam_baseline=False,
):
baseline_span_ala_andrii = Histogram1D(data_shape=(1296, ),
bin_edges=np.arange(0, 50, 1))
random_charge = Histogram1D(data_shape=(1296, ),
bin_edges=np.arange(-50, 1000, 1))
if not use_digicam_baseline:
events = calibration_event_stream(input_file,
pixel_id=pixel_id,
max_events=max_events)
raw_histo = build_raw_data_histogram(events)
save_container(raw_histo, output_file, 'histo', 'raw_lsb')
events = calibration_event_stream(input_file,
max_events=max_events,
pixel_id=pixel_id)
events = fill_histogram(events, 0, raw_histo)
events = fill_electronic_baseline(events)
events = subtract_baseline(events)
else:
events = calibration_event_stream(input_file,
max_events=max_events,
pixel_id=pixel_id)
events = subtract_digicam_baseline(events)
events = fill_baseline_span_ala_andrii(events, baseline_span_ala_andrii)
events = fill_random_charge(events, random_charge, 5)
for _ in events:
pass
save_container(
CalibrationHistogramContainer().from_histogram(
baseline_span_ala_andrii), output_file, 'histo',
'baseline_span_ala_andrii')
save_container(
CalibrationHistogramContainer().from_histogram(random_charge),
output_file, 'histo', 'random_charge')
def compute_baseline_histogram(files,
filename,
max_events=None,
pixel_id=None,
event_types=None,
disable_bar=False):
if os.path.exists(filename) and len(files) == 0:
baseline_histo = Histogram1D.load(filename)
return baseline_histo
else:
if pixel_id is None:
pixel_id = convert_pixel_args(None)
n_pixels = len(pixel_id)
events = calibration_event_stream(files,
pixel_id=pixel_id,
max_events=max_events,
disable_bar=disable_bar)
baseline_histo = Histogram1D(
data_shape=(n_pixels, ),
bin_edges=np.arange(0, 4096, 1 / 16),
)
for event in events:
if event_types and event.event_type not in event_types:
continue
baseline_histo.fill(event.data.digicam_baseline.reshape(-1, 1))
baseline_histo.save(filename)
return baseline_histo
def test_calibration_event_stream():
max_events = 10
calib_stream = calibration_event_stream(example_file_path,
max_events=max_events)
values = []
for event_calib in calib_stream:
values.append([
event_calib.data.adc_samples, event_calib.data.digicam_baseline,
event_calib.pixel_id
])
assert len(values) == max_events
del calib_stream
obs_stream = event_stream(example_file_path, max_events=max_events)
for i, event in enumerate(obs_stream):
event = list(event.r0.tel.values())[0]
assert (values[i][0] == event.adc_samples).all()
assert (values[i][1] == event.digicam_baseline).all()
assert len(values[i][2]) == event.adc_samples.shape[0]
def compute_max_histo(files, histo_filename, pixel_id, max_events,
integral_width, shift, baseline):
n_pixels = len(pixel_id)
if not os.path.exists(histo_filename):
events = calibration_event_stream(files,
pixel_id=pixel_id,
max_events=max_events)
# events = compute_baseline_with_min(events)
events = fill_baseline(events, baseline)
events = subtract_baseline(events)
events = find_pulse_with_max(events)
events = compute_charge(events, integral_width, shift)
max_histo = Histogram1D(
data_shape=(n_pixels, ),
bin_edges=np.arange(-4095 * integral_width, 4095 * integral_width),
)
for event in events:
max_histo.fill(event.data.reconstructed_charge)
max_histo.save(histo_filename)
return max_histo
else:
max_histo = Histogram1D.load(histo_filename)
return max_histo
def compute(files,
max_events,
pixel_id,
output_path,
n_samples,
filename='timing_histo.pk',
save=True,
time_method=compute_time_from_max):
filename = os.path.join(output_path, filename)
if os.path.exists(filename) and save:
raise IOError('The file {} already exists \n'.format(filename))
n_pixels = len(pixel_id)
events = calibration_event_stream(files,
pixel_id=pixel_id,
max_events=max_events)
events = time_method(events)
timing_histo = Histogram1D(data_shape=(n_pixels, ),
bin_edges=np.arange(0, n_samples * 4, 1),
axis_name='reconstructed time [ns]')
for i, event in enumerate(events):
timing_histo.fill(event.data.reconstructed_time)
if save:
timing_histo.save(filename)
return timing_histo
def test_calibration_event_stream():
events = calibration_event_stream([example_file_path])
for event in events:
event_id = event.event_id
energy = event.mc.energy
break
assert event_id == EVENT_ID
assert energy == ENERGY
def compute(files,
pixel_id,
max_events,
pulse_indices,
integral_width,
shift,
bin_width,
output_path,
charge_histo_filename='charge_histo.pk',
amplitude_histo_filename='amplitude_histo.pk',
save=True):
amplitude_histo_path = os.path.join(output_path, amplitude_histo_filename)
charge_histo_path = os.path.join(output_path, charge_histo_filename)
if os.path.exists(charge_histo_path) and save:
raise IOError('File {} already exists'.format(charge_histo_path))
if os.path.exists(amplitude_histo_path) and save:
raise IOError('File {} already exists'.format(amplitude_histo_path))
n_pixels = len(pixel_id)
events = calibration_event_stream(files,
pixel_id=pixel_id,
max_events=max_events)
# events = compute_baseline_with_min(events)
events = fill_digicam_baseline(events)
events = subtract_baseline(events)
# events = find_pulse_with_max(events)
events = fill_pulse_indices(events, pulse_indices)
events = compute_charge(events, integral_width, shift)
events = compute_amplitude(events)
charge_histo = Histogram1D(data_shape=(n_pixels, ),
bin_edges=np.arange(-4095 * integral_width,
4096 * integral_width,
bin_width),
axis_name='reconstructed charge '
'[LSB $\cdot$ ns]')
amplitude_histo = Histogram1D(data_shape=(n_pixels, ),
bin_edges=np.arange(-4095, 4096, 1),
axis_name='reconstructed amplitude '
'[LSB]')
for event in events:
charge_histo.fill(event.data.reconstructed_charge)
amplitude_histo.fill(event.data.reconstructed_amplitude)
if save:
charge_histo.save(charge_histo_path)
amplitude_histo.save(amplitude_histo_path)
return amplitude_histo, charge_histo
def compute_spe(files,
histo_filename,
pixel_id,
baseline,
max_events,
integral_width,
shift,
pulse_finder_threshold,
debug=False):
if not os.path.exists(histo_filename):
n_pixels = len(pixel_id)
events = calibration_event_stream(files,
max_events=max_events,
pixel_id=pixel_id)
events = fill_baseline(events, baseline)
events = subtract_baseline(events)
# events = find_pulse_1(events, 0.5, 20)
# events = find_pulse_2(events, widths=[5, 6], threshold_sigma=2)
events = find_pulse_fast(events, threshold=pulse_finder_threshold)
# events = find_pulse_fast_2(events, threshold=pulse_finder_threshold,
# min_dist=3)
# events = find_pulse_correlate(events,
# threshold=pulse_finder_threshold)
# events = find_pulse_gaussian_filter(events,
# threshold=pulse_finder_threshold)
# events = find_pulse_wavelets(events, widths=[4, 5, 6],
# threshold_sigma=2)
events = compute_charge(events,
integral_width=integral_width,
shift=shift)
# events = compute_amplitude(events)
# events = fit_template(events)
# events = compute_full_waveform_charge(events)
spe_histo = Histogram1D(data_shape=(n_pixels, ),
bin_edges=np.arange(-4095 * 50, 4095 * 50))
for event in events:
spe_histo.fill(event.data.reconstructed_charge)
spe_histo.save(histo_filename)
return spe_histo
else:
spe_histo = Histogram1D.load(histo_filename)
return spe_histo
def test_add_slow_data_calibration():
data_stream = calibration_event_stream(example_file_path, max_events=100)
data_stream = add_slow_data_calibration(data_stream, basepath=aux_basepath)
ts_slow = []
ts_data = []
for event in data_stream:
ts_slow.append(event.slow_data.DriveSystem.timestamp * 1e-3)
ts_data.append(event.data.local_time * 1e-9)
ts_slow = np.array(ts_slow)
ts_data = np.array(ts_data)
diff = ts_data - ts_slow
assert (diff <= 1.1).all()
def compute(files,
max_events,
pixel_id,
n_samples,
ac_levels,
filename='timing_histo.pk',
save=True,
time_method=compute_time_from_max):
if os.path.exists(filename) and save:
raise IOError('The file {} already exists \n'.format(filename))
elif os.path.exists(filename):
return Histogram1D.load(filename)
n_pixels = len(pixel_id)
n_ac_levels = len(ac_levels)
n_files = len(files)
if n_ac_levels != n_files:
raise IOError('Number of files = {} does not match number of DAC level'
' {}'.format(n_files, n_ac_levels))
timing_histo = Histogram1D(
data_shape=(
n_ac_levels,
n_pixels,
),
bin_edges=np.arange(0, n_samples * 4, 1),
)
for i, file in tqdm(enumerate(files), total=n_ac_levels, desc='DAC level'):
events = calibration_event_stream(file,
pixel_id=pixel_id,
max_events=max_events)
events = time_method(events)
for event in events:
timing_histo.fill(event.data.reconstructed_time, indices=(i, ))
if save:
timing_histo.save(filename)
return timing_histo
def compute(files, max_events, filename):
if os.path.exists(filename) and len(files) == 0:
dt_histo = Histogram1D.load(filename)
return dt_histo
else:
events = calibration_event_stream(files, max_events=max_events)
dt_histo = Histogram1D(
data_shape=(9, ),
bin_edges=np.logspace(2, 9, 140), # in ns
# bin_edges=np.arange(150, 400, 4), # in ns
)
previous_time = np.zeros(9) * np.nan
for event in events:
typ = event.event_type
local_time = event.data.local_time # in ns
dt = local_time - previous_time[typ] # in ns
if np.isfinite(previous_time[typ]):
dt_histo.fill(dt, indices=(event.event_type, ))
previous_time[typ] = local_time
dt_histo.save(filename)
print(filename, 'saved')
return dt_histo
def compute(files,
filename,
max_events=None,
pixel_id=None,
event_types=None,
disable_bar=False,
baseline_subtracted=False):
if os.path.exists(filename) and len(files) == 0:
raw_histo = Histogram1D.load(filename)
return raw_histo
else:
if pixel_id is None:
pixel_id = convert_pixel_args(None)
n_pixels = len(pixel_id)
events = calibration_event_stream(files,
pixel_id=pixel_id,
max_events=max_events,
disable_bar=disable_bar)
if baseline_subtracted:
bin_edges = np.arange(-100, 4095, 1)
else:
bin_edges = np.arange(0, 4095, 1)
raw_histo = Histogram1D(
data_shape=(n_pixels, ),
bin_edges=bin_edges,
)
for event in events:
if event_types and event.event_type not in event_types:
continue
samples = event.data.adc_samples
if baseline_subtracted:
samples = samples - event.data.digicam_baseline[:, None]
raw_histo.fill(samples)
raw_histo.save(filename)
return raw_histo
def main(outfile_path, input_files=[]):
events = calibration_event_stream(input_files)
Rough_factor_between_single_pe_amplitude_and_integral = 21 / 5.8
histo = None
for e in tqdm(events):
adc = e.data.adc_samples
adc = adc - e.data.digicam_baseline[:, None]
# I just integrate between sample 10 and 30 to normalize a bit
# normalizing to maximum_amplitude = 1 is too "sharp"
integral = adc[:, 10:30].sum(axis=1)
# handling special case .. we say negative integrals make no sense
# and zero integral simply means there was no pulse at all.
# so we clip at 1
integral = integral.clip(1)
adc = (adc / integral[:, None]
) * Rough_factor_between_single_pe_amplitude_and_integral
arrival_time_in_ns = estimate_arrival_time(adc) * 4
time_in_ns = np.arange(adc.shape[1]) * 4
# TODO: Would be nice to move this out of the loop
if histo is None:
histo = Histogram2dChunked(shape=(adc.shape[0], 101, 101),
range=[[-10, 40], [-0.2, 1.5]])
histo.fill(x=time_in_ns[None, :] - arrival_time_in_ns[:, None], y=adc)
outfile = h5py.File(outfile_path)
dset = outfile.create_dataset(name='adc_count_histo',
data=histo.contents(),
compression='gzip')
dset.attrs['extent'] = histo.extent
def entry():
args = docopt(__doc__)
files = args['<INPUT>']
max_events = convert_int(args['--max_events'])
output_path = args['--output']
if not os.path.exists(output_path):
raise IOError('Path {} for output does not '
'exists \n'.format(output_path))
pixel_id = convert_pixel_args(args['--pixel'])
dc_levels = convert_list_int(args['--dc_levels'])
n_pixels = len(pixel_id)
n_dc_levels = len(dc_levels)
results_filename = 'baseline_shift_results.npz'
results_filename = os.path.join(output_path, results_filename)
# fmpe_results_filename = args['--gain']
# crosstalk = args['--crosstalk']
# templates = args['--template']
histo = Histogram1D(data_shape=(n_dc_levels, n_pixels),
bin_edges=np.arange(0, 4096))
if args['--compute']:
if n_dc_levels != len(files):
raise ValueError('n_dc levels = {} != '
'n_files = {}'.format(n_dc_levels, len(files)))
baseline_mean = np.zeros((n_dc_levels, n_pixels))
baseline_std = np.zeros((n_dc_levels, n_pixels))
for i, file in tqdm(enumerate(files),
desc='DC level',
total=len(files)):
events = calibration_event_stream(file,
pixel_id=pixel_id,
max_events=max_events)
for count, event in enumerate(events):
baseline_mean[i] += event.data.digicam_baseline
baseline_std[i] += event.data.digicam_baseline**2
histo.fill(event.data.adc_samples, indices=(i, ))
count += 1
baseline_mean[i] = baseline_mean[i] / count
baseline_std[i] = baseline_std[i] / count
baseline_std[i] = baseline_std[i] - baseline_mean[i]**2
baseline_std[i] = np.sqrt(baseline_std[i])
histo.save(os.path.join(output_path, 'raw_histo.pk'))
np.savez(results_filename,
baseline_mean=baseline_mean,
baseline_std=baseline_std,
dc_levels=dc_levels)
if args['--fit']:
data = dict(np.load(results_filename))
baseline_mean = data['baseline_mean']
baseline_std = data['baseline_std']
dc_levels = data['dc_levels']
gain = 5
template_area = 18
crosstalk = 0.08
bias_resistance = 10 * 1E3
cell_capacitance = 50 * 1E-15
baseline_shift = baseline_mean - baseline_mean[0]
nsb_rate = gain * template_area / (baseline_shift * (1 - crosstalk))
nsb_rate = nsb_rate - cell_capacitance * bias_resistance
nsb_rate = 1 / nsb_rate
np.savez(results_filename,
baseline_mean=baseline_mean,
baseline_std=baseline_std,
dc_levels=dc_levels,
nsb_rate=nsb_rate,
baseline_shift=baseline_shift)
if args['--save_figures']:
pass
if args['--display']:
data = dict(np.load(results_filename))
histo = Histogram1D.load(os.path.join(output_path, 'raw_histo.pk'))
baseline_mean = histo.mean()
baseline_std = histo.std()
nsb_rate = data['nsb_rate']
print(baseline_mean.shape)
histo.draw((0, 1))
histo.draw((49, 1))
plt.figure()
plt.plot(dc_levels, baseline_mean)
plt.xlabel('DC DAC level')
plt.ylabel('Baseline mean [LSB]')
plt.figure()
plt.plot(dc_levels, baseline_std)
plt.xlabel('DC DAC level')
plt.ylabel('Baseline std [LSB]')
plt.figure()
plt.plot(nsb_rate, baseline_std)
plt.xlabel('$f_{NSB}$ [GHz]')
plt.ylabel('Baseline std [LSB]')
plt.figure()
plt.plot(baseline_mean, baseline_std)
plt.xlabel('Baseline mean [LSB]')
plt.ylabel('Baseline std [LSB]')
plt.figure()
plt.semilogy(dc_levels, nsb_rate)
plt.xlabel('DC DAC level')
plt.ylabel('$f_{NSB}$ [GHz]')
plt.show()
pass
return
def main(
input_files,
output_hist,
delays_ns=None,
time_range_ns=(-10., 40.),
amplitude_range=(-0.1, 0.4),
integration_range=None,
# charge < 50 pe (noisy) or > 500 pe (saturation) => bad_charge
# 1 pe <=> 20 LSB integral
charge_range=(1000., 8000.),
n_bin=100,
disable_bar=False):
if delays_ns is not None:
assert len(delays_ns) == len(input_files)
charge_min = np.min(charge_range)
charge_max = np.max(charge_range)
if integration_range is not None:
integration_min = np.min(integration_range)
integration_max = np.max(integration_range)
histo = None
n_sample = 0
n_pixel = 0
for file_idx, input_file in enumerate(input_files):
if not os.path.isfile(input_file):
continue
events = calibration_event_stream([input_file],
disable_bar=disable_bar)
events = fill_digicam_baseline(events)
if "SST1M_01_201805" in input_files[0]: # fix data in May
print("WARNING: correction of the baselines applied.")
events = correct_wrong_baseline(events)
events = subtract_baseline(events)
for e in events:
adc = e.data.adc_samples
if integration_range is not None:
adc_interp = adc[:, slice(integration_min, integration_max)]
else:
adc_interp = adc
integral = adc_interp.sum(axis=1)
adc_norm = adc / integral[:, None]
if delays_ns is None:
arrival_time_in_ns = estimate_time_from_leading_edge(adc) * 4
else:
arrival_time_in_ns = delays_ns[file_idx] * np.ones(1296)
if histo is None:
n_pixel, n_sample = adc_norm.shape
histo = Histogram2dChunked(
shape=(n_pixel, n_bin, n_bin),
range=[time_range_ns, amplitude_range])
else:
assert adc_norm.shape[0] == n_pixel
assert adc_norm.shape[1] == n_sample
time_in_ns = np.arange(n_sample) * 4
bad_charge = np.logical_or(integral < charge_min,
integral > charge_max)
arrival_time_in_ns[bad_charge] = -np.inf # ignored by histo
histo.fill(x=time_in_ns[None, :] - arrival_time_in_ns[:, None],
y=adc_norm)
if os.path.exists(output_hist):
os.remove(output_hist)
histo.save(output_hist)
print('2D histogram of pulse shape for all pixel saved as', output_hist)
pe_mean = np.zeros(shape)
pe_std = np.zeros(shape)
timing = np.load('/sst1m/analyzed/calib/timing/timing.npz')
timing = timing['time'] // 4
data_1 = dict(np.load(filename_1_dark))
dark_baseline = data_1['baseline_mean'][0]
charge_mean = data_1['charge_mean']
print(dark_baseline)
pe_interpolator = lambda x: charge_to_pe(x, charge_mean, true_pe)
for i, file in tqdm(enumerate(files), total=n_files):
events = calibration_event_stream(file, max_events=max_events)
events = fill_dark_baseline(events, dark_baseline)
events = fill_digicam_baseline(events)
events = compute_baseline_shift(events)
events = subtract_baseline(events)
# events = compute_nsb_rate(events, gain, pulse_area, crosstalk,
# bias_resistance, cell_capacitance)
# events = compute_charge_with_saturation(events, integral_width=7)
events = compute_charge_with_saturation_and_threshold(events,
integral_width=integral_width,
debug=debug,
trigger_bin=timing,
saturation_threshold=saturation_threshold,
pulse_tail=pulse_tail)
events = compute_number_of_pe_from_table(events, pe_interpolator)
def data_quality(
files, dark_filename, time_step, fits_filename, load_files,
histo_filename, rate_plot_filename, baseline_plot_filename,
nsb_plot_filename, parameters_filename, template_filename,
aux_basepath, threshold_sample_pe=20.,
bias_resistance=1e4 * u.Ohm, cell_capacitance=5e-14 * u.Farad,
disable_bar=False, aux_services=('DriveSystem',)
):
input_dir = np.unique([os.path.dirname(file) for file in files])
if len(input_dir) > 1:
raise AttributeError("input files must be from the same directories")
input_dir = input_dir[0]
if aux_basepath.lower() == "search":
aux_basepath = input_dir.replace('/raw/', '/aux/')
print("auxiliary files are expected in", aux_basepath)
with open(parameters_filename) as file:
calibration_parameters = yaml.load(file)
pulse_template = NormalizedPulseTemplate.load(template_filename)
pulse_area = pulse_template.integral() * u.ns
gain_integral = np.array(calibration_parameters['gain'])
charge_to_amplitude = pulse_template.compute_charge_amplitude_ratio(7, 4)
gain_amplitude = gain_integral * charge_to_amplitude
crosstalk = np.array(calibration_parameters['mu_xt'])
pixel_id = np.arange(1296)
n_pixels = len(pixel_id)
dark_histo = Histogram1D.load(dark_filename)
dark_baseline = dark_histo.mean()
if not load_files:
events = calibration_event_stream(files, disable_bar=disable_bar)
events = add_slow_data_calibration(
events, basepath=aux_basepath, aux_services=aux_services
)
events = fill_digicam_baseline(events)
events = fill_dark_baseline(events, dark_baseline)
events = subtract_baseline(events)
events = compute_baseline_shift(events)
events = compute_nsb_rate(
events, gain_amplitude, pulse_area, crosstalk, bias_resistance,
cell_capacitance
)
events = compute_gain_drop(events, bias_resistance, cell_capacitance)
events = compute_sample_photo_electron(events, gain_amplitude)
events = tag_burst_from_moving_average_baseline(
events, n_previous_events=100, threshold_lsb=5
)
events = compute_3d_cleaning(events, geom=DigiCam.geometry,
threshold_sample_pe=threshold_sample_pe)
init_time = 0
baseline = 0
count = 0
shower_count = 0
az = 0
el = 0
container = DataQualityContainer()
file = Serializer(fits_filename, mode='w', format='fits')
baseline_histo = Histogram1D(
data_shape=(n_pixels,),
bin_edges=np.arange(4096)
)
for i, event in enumerate(events):
new_time = event.data.local_time
if init_time == 0:
init_time = new_time
count += 1
baseline += np.mean(event.data.digicam_baseline)
az += event.slow_data.DriveSystem.current_position_az
el += event.slow_data.DriveSystem.current_position_el
time_diff = new_time - init_time
if event.data.shower:
shower_count += 1
baseline_histo.fill(event.data.digicam_baseline.reshape(-1, 1))
if time_diff > time_step and i > 0:
trigger_rate = count / time_diff
shower_rate = shower_count / time_diff
baseline = baseline / count
az = az / count
el = el / count
container.trigger_rate = trigger_rate
container.baseline = baseline
container.time = (new_time + init_time) / 2
container.shower_rate = shower_rate
container.burst = event.data.burst
nsb_rate = event.data.nsb_rate
container.nsb_rate = np.nanmean(nsb_rate).value
container.current_position_az = az
container.current_position_el = el
baseline = 0
count = 0
init_time = 0
shower_count = 0
az = 0
el = 0
file.add_container(container)
output_path = os.path.dirname(histo_filename)
if not os.path.exists(output_path):
os.makedirs(output_path)
baseline_histo.save(histo_filename)
print(histo_filename, 'created.')
file.close()
print(fits_filename, 'created.')
data = Table.read(fits_filename, format='fits')
data = data.to_pandas()
data['time'] = pd.to_datetime(data['time'], utc=True)
data = data.set_index('time')
if rate_plot_filename is not None:
fig1 = plt.figure()
ax = plt.gca()
plt.xticks(rotation=70)
plt.plot(data['trigger_rate']*1E9, '.', label='trigger rate')
plt.plot(data['shower_rate']*1E9, '.', label='shower_rate')
plt.ylabel('rate [Hz]')
plt.legend({'trigger rate', 'shower rate'})
xlim = plt.xlim()
plt.xlim(xlim[0] - 1e-3, xlim[1] + 1e-3) # extra min on the sides
if rate_plot_filename == "show":
plt.show()
else:
output_path = os.path.dirname(rate_plot_filename)
if not (output_path == '' or os.path.exists(output_path)):
os.makedirs(output_path)
plt.savefig(rate_plot_filename)
plt.close(fig1)
if baseline_plot_filename is not None:
fig2 = plt.figure(figsize=(8, 6))
ax = plt.gca()
data_burst = data[data['burst']]
data_good = data[~data['burst']]
plt.xticks(rotation=70)
plt.plot(data_good['baseline'], '.', label='good', ms=2)
plt.plot(data_burst['baseline'], '.', label='burst', ms=2)
plt.ylabel('Baseline [LSB]')
xlim = plt.xlim()
plt.xlim(xlim[0] - 1e-3, xlim[1] + 1e-3) # extra min on the sides
if rate_plot_filename == "show":
plt.show()
else:
output_path = os.path.dirname(baseline_plot_filename)
if not (output_path == '' or os.path.exists(output_path)):
os.makedirs(output_path)
plt.savefig(baseline_plot_filename)
plt.close(fig2)
if nsb_plot_filename is not None:
fig3 = plt.figure()
ax = fig3.add_subplot(111)
data.plot(y='nsb_rate', ax=ax)
ax.set_ylabel('$f_{NSB}$ [GHz]')
if nsb_plot_filename == "show":
plt.show()
else:
fig3.savefig(nsb_plot_filename)
plt.close(fig3)
return
def nsb_rate(
files, aux_basepath, dark_histo_file, param_file, template_filename,
output=None, plot="show", plot_nsb_range=None, norm="log",
plot_baselines=False, disable_bar=False, max_events=None, n_skip=10,
stars=True,
bias_resistance=1e4 * u.Ohm, cell_capacitance=5e-14 * u.Farad
):
files = np.atleast_1d(files)
if len(files) == 1 and not files[0].endswith('.fz'):
table = Table.read(files[0])[:max_events]
data = dict(table)
data['nsb_rate'] = np.array(data['nsb_rate']) * u.GHz
else:
dark_histo = Histogram1D.load(dark_histo_file)
n_pixel = len(DigiCam.geometry.neighbors)
pixels = np.arange(n_pixel, dtype=int)
with open(param_file) as file:
pulse_template = NormalizedPulseTemplate.load(template_filename)
pulse_area = pulse_template.integral() * u.ns
charge_to_amplitude = pulse_template.compute_charge_amplitude_ratio(7, 4)
calibration_parameters = yaml.load(file)
gain_integral = np.array(calibration_parameters['gain'])
gain_amplitude = gain_integral * charge_to_amplitude
crosstalk = np.array(calibration_parameters['mu_xt'])
events = calibration_event_stream(files, max_events=max_events,
disable_bar=disable_bar)
events = add_slow_data_calibration(
events, basepath=aux_basepath,
aux_services=('DriveSystem', )
)
data = {
"baseline": [],
"nsb_rate": [],
"good_pixels_mask": [],
"timestamp": [],
"event_id": [],
"az": [],
"el": [],
}
bad_pixels = get_bad_pixels(
calib_file=param_file, nsigma_gain=5, nsigma_elecnoise=5,
dark_histo=dark_histo_file, nsigma_dark=8, plot=None, output=None
)
events_skipped = 0
for event in events:
if event.event_type.INTERNAL not in event.event_type:
continue
events_skipped += 1
if events_skipped < n_skip:
continue
events_skipped = 0
data['baseline'].append(event.data.digicam_baseline)
baseline_shift = event.data.digicam_baseline - dark_histo.mean()
rate = _compute_nsb_rate(
baseline_shift=baseline_shift, gain=gain_amplitude,
pulse_area=pulse_area, crosstalk=crosstalk,
bias_resistance=bias_resistance, cell_capacitance=cell_capacitance
)
bad_pixels_event = np.unique(np.hstack(
(
bad_pixels,
pixels[rate < 0],
pixels[rate > 5 * u.GHz]
)
))
avg_matrix = _get_average_matrix_bad_pixels(
DigiCam.geometry, bad_pixels_event
)
good_pixels_mask = np.ones(n_pixel, dtype=bool)
good_pixels_mask[bad_pixels_event] = False
good_pixels = pixels[good_pixels_mask]
rate[bad_pixels_event] = avg_matrix[bad_pixels_event, :].dot(
rate[good_pixels]
)
data['good_pixels_mask'].append(good_pixels_mask)
data['timestamp'].append(event.data.local_time)
data['event_id'].append(event.event_id)
data['nsb_rate'].append(rate)
data['az'].append(event.slow_data.DriveSystem.current_position_az)
data['el'].append(event.slow_data.DriveSystem.current_position_el)
data['nsb_rate'] = np.array(data['nsb_rate']) * u.GHz
if output is not None:
table = Table(data)
if os.path.isfile(output):
os.remove(output)
table.write(output, format='fits')
time_obs = Time(
np.array(data['timestamp'], dtype=np.float64) * 1e-9,
format='unix'
)
if plot_baselines:
fig2, (ax1, ax2) = plt.subplots(2, 1, figsize=(16, 8), dpi=50)
baseline_std = np.std(data['baseline'], axis=0)
ax1.hist(baseline_std, 100)
ax1.set_xlabel('std(baseline) [LSB]')
# pixels_shown = np.arange(len(baseline_std))[baseline_std > 10]
pixels_shown = [834,]
ax2.plot_date(
time_obs.to_datetime(),
data['baseline'][:, pixels_shown],
'-'
)
ax2.set_xlabel('time')
ax2.set_ylabel('baseline [LSB]')
plt.tight_layout()
plt.show()
plt.close(fig2)
az_obs = np.array(data['az']) * u.deg
el_obs = np.array(data['el']) * u.deg
n_event = len(data['timestamp'])
fig1, ax = plt.subplots(1, 1, figsize=(16, 12), dpi=50)
date = datetime.fromtimestamp(data['timestamp'][0]*1e-9)
date_str = date.strftime("%H:%M:%S")
display = CameraDisplay(
DigiCam.geometry, ax=ax, norm=norm,
title='NSB rate [GHz], t=' + date_str
)
rate_ghz = np.array(data['nsb_rate'][0].to(u.GHz).value)
display.image = rate_ghz
if plot_nsb_range is None:
min_range_rate = np.max([np.min(rate_ghz), 50e-3])
plot_nsb_range = (min_range_rate, np.max(rate_ghz))
display.set_limits_minmax(*plot_nsb_range)
display.add_colorbar(ax=ax)
bad_pixels = np.arange(
len(data['good_pixels_mask'][0])
)[~data['good_pixels_mask'][0]]
display.highlight_pixels(bad_pixels, color='r', linewidth=2)
display.axes.set_xlim([-500., 500.])
display.axes.set_ylim([-500., 500.])
plt.tight_layout()
if stars is True:
stars_az, stars_alt, stars_pmag = get_stars_in_fov(
az_obs[0], el_obs[0], time_obs
)
stars_x, stars_y = transform_azel_to_xy(
stars_az, stars_alt, az_obs, el_obs
)
point_stars = []
for index_star in range(len(stars_pmag)):
point_star, = ax.plot(
stars_x[index_star, 0],
stars_y[index_star, 0],
'ok',
ms=20-2*stars_pmag[index_star],
mew=3,
mfc='None'
)
point_stars.append(point_star)
def update(i, display):
print('frame', i, '/', len(data['timestamp']))
display.image = data['nsb_rate'][i].to(u.GHz).value
date = datetime.fromtimestamp(data['timestamp'][i] * 1e-9)
date_str = date.strftime("%H:%M:%S")
display.axes.set_title('NSB rate [GHz], t=' + date_str)
bad_pixels = np.arange(
len(data['good_pixels_mask'][i])
)[~data['good_pixels_mask'][i]]
display.highlight_pixels(
bad_pixels, color='r', linewidth=2
)
if stars is True:
for index_star in range(len(stars_pmag)):
point_stars[index_star].set_xdata(
stars_x[index_star, i]
)
point_stars[index_star].set_ydata(
stars_y[index_star, i]
)
anim = FuncAnimation(
fig1,
update,
frames=len(data['timestamp']),
interval=20,
fargs=(display, )
)
Writer = animation.writers['ffmpeg']
writer = Writer(fps=50, metadata=dict(artist='Y. Renier'),
bitrate=4000, codec='h263p')
output_path = os.path.dirname(plot)
if plot == "show" or \
(output_path != "" and not os.path.isdir(output_path)):
if not plot == "show":
print('WARNING: Path ' + output_path + ' for output trigger ' +
'uniformity does not exist, displaying the plot instead.\n')
display.enable_pixel_picker()
plt.show()
else:
anim.save(plot, writer=writer)
print(plot, 'created')
plt.close(fig1)
def compute(files, ac_levels, dc_levels, output_filename, dark_charge, dark_baseline,
max_events, pixels, integral_width, timing, saturation_threshold, pulse_tail, debug):
directory = '/sst1m/analyzed/calib/mpe/'
file_calib = os.path.join(directory, 'mpe_fit_results_combined.npz')
data_calib = dict(np.load(file_calib))
pe = data_calib['mu']
pe_err = data_calib['mu_error']
ac = data_calib['ac_levels'][:, 0]
ac_led = ACLED(ac, pe, pe_err)
pde = 0.9 # window filter
true_pe = ac_led(ac_levels).T * pde
# mask = true_pe < 5
# true_pe[mask] = pe[mask]
n_pixels = len(pixels)
n_ac_level = len(ac_levels)
n_dc_level = len(dc_levels)
n_files = len(files)
assert n_files == (n_ac_level * n_dc_level)
debug = False
pulse_tail = False
shape = (n_dc_level, n_ac_level, n_pixels)
nsb_mean = np.zeros(shape)
nsb_std = np.zeros(shape)
pe_mean = np.zeros(shape)
pe_std = np.zeros(shape)
print(dark_baseline, dark_charge)
pe_interpolator = lambda x: charge_to_pe(x, dark_charge, true_pe)
for i, dc_level, in tqdm(enumerate(dc_levels), total=n_dc_level):
for j, ac_level in tqdm(enumerate(ac_levels), total=n_ac_level):
index_file = i * n_ac_level + j
file = files[index_file]
events = calibration_event_stream(file, max_events=max_events)
events = fill_dark_baseline(events, dark_baseline)
events = fill_digicam_baseline(events)
events = compute_baseline_shift(events)
events = subtract_baseline(events)
# events = compute_nsb_rate(events, gain, pulse_area, crosstalk,
# bias_resistance, cell_capacitance)
# events = compute_charge_with_saturation(events, integral_width=7)
events = compute_charge_with_saturation_and_threshold(events,
integral_width=integral_width,
debug=debug,
trigger_bin=timing,
saturation_threshold=saturation_threshold,
pulse_tail=pulse_tail)
events = compute_number_of_pe_from_table(events, pe_interpolator)
events = rescale_pulse(events, gain_func=_gain_drop_from_baseline_shift,
xt_func=_crosstalk_drop_from_baseline_shift,
pde_func=_pde_drop_from_baseline_shift)
# events = compute_maximal_charge(events)
for n, event in enumerate(events):
pe_mean[i, j] += event.data.reconstructed_number_of_pe
pe_std[i, j] += event.data.reconstructed_number_of_pe**2
# nsb_mean[i] += event.data.nsb_rate
# nsb_std[i] += event.data.nsb_rate**2
# print(event.data.baseline_shift)
pe_mean[i, j] = pe_mean[i, j] / (n + 1)
# nsb_mean[i] = nsb_mean[i] / (n + 1)
pe_std[i, j] = pe_std[i, j] / (n + 1)
pe_std[i, j] = np.sqrt(pe_std[i, j] - pe_mean[i, j]**2)
# nsb_std[i] = nsb_std[i] / (n + 1)
# nsb_std[i] = np.sqrt(nsb_std[i] - nsb_mean[i]**2)
np.savez(output_filename, pe_reco_mean=pe_mean, pe_reco_std=pe_std,
ac_levels=ac_levels, pe=pe, pe_err=pe_err, true_pe=true_pe,
nsb_mean=nsb_mean, nsb_std=nsb_std)
def entry():
args = docopt(__doc__)
files = args['INPUT']
debug = args['--debug']
max_events = convert_max_events_args(args['--max_events'])
output_path = args['OUTPUT']
if not os.path.exists(output_path):
raise IOError('Path for output does not exists \n')
pixel_id = convert_pixel_args(args['--pixel'])
n_pixels = len(pixel_id)
raw_histo_filename = 'raw_histo.pk'
amplitude_histo_filename = output_path + 'amplitude_histo.pk'
charge_histo_filename = output_path + 'charge_histo.pk'
max_histo_filename = output_path + 'max_histo.pk'
results_filename = output_path + 'fit_results.h5'
dark_count_rate_filename = output_path + 'dark_count_rate.npz'
crosstalk_filename = output_path + 'crosstalk.npz'
electronic_noise_filename = output_path + 'electronic_noise.npz'
integral_width = int(args['--integral_width'])
shift = int(args['--shift'])
pulse_finder_threshold = float(args['--pulse_finder_threshold'])
n_samples = int(args['--n_samples']) # TODO access this in a better way !
if args['--compute']:
raw_histo = raw.compute(files,
max_events=max_events,
pixel_id=pixel_id,
output_path=output_path,
filename=raw_histo_filename)
baseline = raw_histo.mode()
events = calibration_event_stream(files,
pixel_id=pixel_id,
max_events=max_events)
# events = compute_baseline_with_min(events)
events = fill_baseline(events, baseline)
events = subtract_baseline(events)
events = find_pulse_with_max(events)
events = compute_charge(events, integral_width, shift)
max_histo = Histogram1D(data_shape=(n_pixels, ),
bin_edges=np.arange(-4095 * integral_width,
4095 * integral_width),
axis_name='[LSB]')
for event in events:
max_histo.fill(event.data.reconstructed_charge)
max_histo.save(max_histo_filename)
events = calibration_event_stream(files,
max_events=max_events,
pixel_id=pixel_id)
events = fill_baseline(events, baseline)
events = subtract_baseline(events)
# events = find_pulse_1(events, 0.5, 20)
# events = find_pulse_2(events, widths=[5, 6], threshold_sigma=2)
# events = find_pulse_fast(events, threshold=pulse_finder_threshold)
# events = find_pulse_correlate(events,
# threshold=pulse_finder_threshold)
# events = find_pulse_gaussian_filter(events,
# threshold=pulse_finder_threshold)
events = find_pulse_wavelets(events,
widths=[4, 5, 6],
threshold_sigma=2)
events = compute_charge(events,
integral_width=integral_width,
shift=shift)
events = compute_amplitude(events)
# events = fit_template(events)
if debug:
events = plot_event(events, 0)
spe_charge = Histogram1D(data_shape=(n_pixels, ),
bin_edges=np.arange(-4095 * integral_width,
4095 * integral_width))
spe_amplitude = Histogram1D(data_shape=(n_pixels, ),
bin_edges=np.arange(-4095, 4095, 1))
for event in events:
spe_charge.fill(event.data.reconstructed_charge)
spe_amplitude.fill(event.data.reconstructed_amplitude)
spe_charge.save(charge_histo_filename)
spe_amplitude.save(amplitude_histo_filename)
if args['--fit']:
spe_charge = Histogram1D.load(charge_histo_filename)
spe_amplitude = Histogram1D.load(amplitude_histo_filename)
max_histo = Histogram1D.load(max_histo_filename)
dark_count_rate = np.zeros(n_pixels) * np.nan
electronic_noise = np.zeros(n_pixels) * np.nan
for i, pixel in tqdm(enumerate(pixel_id), total=n_pixels,
desc='Pixel'):
x = max_histo._bin_centers()
y = max_histo.data[i]
n_entries = np.sum(y)
mask = (y > 0)
x = x[mask]
y = y[mask]
try:
val, err = compute_gaussian_parameters_first_peak(
x,
y,
snr=3,
)
number_of_zeros = val['amplitude']
window_length = 4 * n_samples
rate = compute_dark_rate(number_of_zeros, n_entries,
window_length)
dark_count_rate[pixel] = rate
electronic_noise[pixel] = val['sigma']
except Exception as e:
print('Could not compute dark count rate in pixel {}'.format(
pixel))
print(e)
np.savez(dark_count_rate_filename, dcr=dark_count_rate)
np.savez(electronic_noise_filename, electronic_noise)
spe = spe_charge
name = 'charge'
crosstalk = np.zeros(n_pixels) * np.nan
results = SPEResultContainer()
table_name = 'analysis_' + name
with HDF5TableWriter(results_filename, table_name, mode='w') as h5:
for i, pixel in tqdm(enumerate(pixel_id),
total=n_pixels,
desc='Pixel'):
try:
x = spe._bin_centers()
y = spe.data[i]
y_err = spe.errors(index=i)
sigma_e = electronic_noise[i]
sigma_e = sigma_e if not np.isnan(sigma_e) else None
params, params_err, params_init, params_bound = \
fit_spe(x, y, y_err, sigma_e=sigma_e,
snr=3, debug=debug)
for key, val in params.items():
setattr(results.init, key, params_init[key])
setattr(results.param, key, params[key])
setattr(results.param_errors, key, params_err[key])
for key, val in results.items():
results[key]['pixel_id'] = pixel
for key, val in results.items():
h5.write('spe_' + key, val)
n_entries = params['a_1']
n_entries += params['a_2']
n_entries += params['a_3']
n_entries += params['a_4']
crosstalk[pixel] = (n_entries - params['a_1']) / n_entries
except Exception as e:
print('Could not fit for pixel_id : {}'.format(pixel))
print(e)
np.savez(crosstalk_filename, crosstalk)
if args['--save_figures']:
spe_charge = Histogram1D.load(charge_histo_filename)
spe_amplitude = Histogram1D.load(amplitude_histo_filename)
raw_histo = Histogram1D.load(
os.path.join(output_path, raw_histo_filename))
max_histo = Histogram1D.load(max_histo_filename)
figure_directory = output_path + 'figures/'
if not os.path.exists(figure_directory):
os.makedirs(figure_directory)
histograms = [spe_charge, spe_amplitude, raw_histo, max_histo]
names = [
'histogram_charge/', 'histogram_amplitude/', 'histogram_raw/',
'histo_max/'
]
for i, histo in enumerate(histograms):
figure = plt.figure()
histogram_figure_directory = figure_directory + names[i]
if not os.path.exists(histogram_figure_directory):
os.makedirs(histogram_figure_directory)
for j, pixel in enumerate(pixel_id):
axis = figure.add_subplot(111)
figure_path = histogram_figure_directory + 'pixel_{}'. \
format(pixel)
try:
histo.draw(index=(j, ), axis=axis, log=True, legend=False)
figure.savefig(figure_path)
except Exception as e:
print('Could not save pixel {} to : {} \n'.format(
pixel, figure_path))
print(e)
axis.remove()
if args['--display']:
spe_charge = Histogram1D.load(charge_histo_filename)
spe_amplitude = Histogram1D.load(amplitude_histo_filename)
raw_histo = Histogram1D.load(
os.path.join(output_path, raw_histo_filename))
max_histo = Histogram1D.load(max_histo_filename)
spe_charge.draw(index=(0, ), log=True, legend=False)
spe_amplitude.draw(index=(0, ), log=True, legend=False)
raw_histo.draw(index=(0, ), log=True, legend=False)
max_histo.draw(index=(0, ), log=True, legend=False)
try:
df = pd.HDFStore(results_filename, mode='r')
parameters = df['analysis_charge/spe_param']
parameters_error = df['analysis_charge/spe_param_errors']
dark_count_rate = np.load(dark_count_rate_filename)['dcr']
crosstalk = np.load(crosstalk_filename)['arr_0']
except IOError as e:
print(e)
print('Could not find the analysis files !')
plt.show()
exit()
label = 'mean : {:2f} \n std : {:2f}'
for key, val in parameters.items():
fig = plt.figure()
axes = fig.add_subplot(111)
axes.hist(val,
bins='auto',
label=label.format(np.mean(val), np.std(val)))
axes.set_xlabel(key + ' []')
axes.set_ylabel('count []')
axes.legend(loc='best')
# fig = plt.figure()
# axes = fig.add_subplot(111)
# axes.hist(parameters_error[key])
# axes.set_xlabel(key + '_error' + ' []')
# axes.set_ylabel('count []')
crosstalk = crosstalk[np.isfinite(crosstalk)]
dark_count_rate = dark_count_rate[np.isfinite(dark_count_rate)]
plt.figure()
plt.hist(crosstalk,
bins='auto',
label=label.format(np.mean(crosstalk), np.std(crosstalk)))
plt.xlabel('XT []')
plt.legend(loc='best')
plt.figure()
plt.hist(dark_count_rate[np.isfinite(dark_count_rate)],
bins='auto',
label=label.format(np.mean(dark_count_rate),
np.std(dark_count_rate)))
plt.xlabel('dark count rate [GHz]')
plt.legend(loc='best')
plt.show()
return
def compute(files,
pixel_id,
max_events,
pulse_indices,
integral_width,
shift,
bin_width,
charge_histo_filename='charge_histo.pk',
amplitude_histo_filename='amplitude_histo.pk',
save=True):
if os.path.exists(charge_histo_filename) and save:
raise IOError('File {} already exists'.format(charge_histo_filename))
elif os.path.exists(charge_histo_filename):
charge_histo = Histogram1D.load(charge_histo_filename)
if os.path.exists(amplitude_histo_filename) and save:
raise IOError(
'File {} already exists'.format(amplitude_histo_filename))
elif os.path.exists(amplitude_histo_filename):
amplitude_histo = Histogram1D.load(amplitude_histo_filename)
if (not os.path.exists(amplitude_histo_filename)) or \
(not os.path.exists(charge_histo_filename)):
n_pixels = len(pixel_id)
events = calibration_event_stream(files,
pixel_id=pixel_id,
max_events=max_events)
# events = compute_baseline_with_min(events)
events = fill_digicam_baseline(events)
events = subtract_baseline(events)
# events = find_pulse_with_max(events)
events = fill_pulse_indices(events, pulse_indices)
events = compute_charge(events, integral_width, shift)
events = compute_amplitude(events)
charge_histo = Histogram1D(data_shape=(n_pixels, ),
bin_edges=np.arange(-40 * integral_width,
4096 * integral_width,
bin_width))
amplitude_histo = Histogram1D(data_shape=(n_pixels, ),
bin_edges=np.arange(-40, 4096, 1))
for event in events:
charge_histo.fill(event.data.reconstructed_charge)
amplitude_histo.fill(event.data.reconstructed_amplitude)
if save:
charge_histo.save(charge_histo_filename)
amplitude_histo.save(amplitude_histo_filename)
return amplitude_histo, charge_histo
def entry():
args = docopt(__doc__)
files = args['<INPUT>']
debug = args['--debug']
max_events = convert_int(args['--max_events'])
results_filename = args['--fit_output']
dir_output = os.path.dirname(results_filename)
if not os.path.exists(dir_output):
raise IOError('Path {} for output '
'does not exists \n'.format(dir_output))
pixel_ids = convert_pixel_args(args['--pixel'])
integral_width = int(args['--integral_width'])
shift = int(args['--shift'])
bin_width = int(args['--bin_width'])
ncall = int(args['--ncall'])
ac_levels = convert_list_int(args['--ac_levels'])
n_pixels = len(pixel_ids)
n_ac_levels = len(ac_levels)
adc_min = int(args['--adc_min'])
adc_max = int(args['--adc_max'])
timing_filename = args['--timing']
timing = np.load(timing_filename)['time']
charge_histo_filename = args['--compute_output']
fmpe_results_filename = args['--gain']
if args['--compute']:
if n_ac_levels != len(files):
raise ValueError('n_ac_levels = {} != '
'n_files = {}'.format(n_ac_levels, len(files)))
time = np.zeros((n_ac_levels, n_pixels))
charge_histo = Histogram1D(bin_edges=np.arange(
adc_min * integral_width, adc_max * integral_width, bin_width),
data_shape=(
n_ac_levels,
n_pixels,
))
if os.path.exists(charge_histo_filename):
raise IOError(
'File {} already exists'.format(charge_histo_filename))
for i, (file, ac_level) in tqdm(enumerate(zip(files, ac_levels)),
total=n_ac_levels,
desc='DAC level',
leave=False):
time[i] = timing[pixel_ids]
pulse_indices = time[i] // 4
events = calibration_event_stream(file,
pixel_id=pixel_ids,
max_events=max_events)
# events = compute_baseline_with_min(events)
events = fill_digicam_baseline(events)
events = subtract_baseline(events)
# events = find_pulse_with_max(events)
events = fill_pulse_indices(events, pulse_indices)
events = compute_charge(events, integral_width, shift)
events = compute_amplitude(events)
for event in events:
charge_histo.fill(event.data.reconstructed_charge, indices=i)
charge_histo.save(charge_histo_filename, )
if args['--fit']:
input_parameters = Table.read(fmpe_results_filename, format='fits')
input_parameters = input_parameters.to_pandas()
gain = np.zeros((n_ac_levels, n_pixels)) * np.nan
sigma_e = np.zeros((n_ac_levels, n_pixels)) * np.nan
sigma_s = np.zeros((n_ac_levels, n_pixels)) * np.nan
baseline = np.zeros((n_ac_levels, n_pixels)) * np.nan
mu = np.zeros((n_ac_levels, n_pixels)) * np.nan
mu_xt = np.zeros((n_ac_levels, n_pixels)) * np.nan
amplitude = np.zeros((n_ac_levels, n_pixels)) * np.nan
gain_error = np.zeros((n_ac_levels, n_pixels)) * np.nan
sigma_e_error = np.zeros((n_ac_levels, n_pixels)) * np.nan
sigma_s_error = np.zeros((n_ac_levels, n_pixels)) * np.nan
baseline_error = np.zeros((n_ac_levels, n_pixels)) * np.nan
mu_error = np.zeros((n_ac_levels, n_pixels)) * np.nan
mu_xt_error = np.zeros((n_ac_levels, n_pixels)) * np.nan
amplitude_error = np.zeros((n_ac_levels, n_pixels)) * np.nan
mean = np.zeros((n_ac_levels, n_pixels)) * np.nan
std = np.zeros((n_ac_levels, n_pixels)) * np.nan
chi_2 = np.zeros((n_ac_levels, n_pixels)) * np.nan
ndf = np.zeros((n_ac_levels, n_pixels)) * np.nan
ac_limit = [np.inf] * n_pixels
charge_histo = Histogram1D.load(charge_histo_filename)
for i, ac_level in tqdm(enumerate(ac_levels),
total=n_ac_levels,
desc='DAC level',
leave=False):
for j, pixel_id in tqdm(enumerate(pixel_ids),
total=n_pixels,
desc='Pixel',
leave=False):
histo = charge_histo[i, pixel_id]
mean[i, j] = histo.mean()
std[i, j] = histo.std()
if histo.overflow > 0 or histo.data.sum() == 0:
continue
fit_params_names = describe(mpe_distribution_general)
options = {'fix_n_peaks': True}
fixed_params = {}
for param in fit_params_names:
if param in input_parameters.keys():
name = 'fix_' + param
options[name] = True
fixed_params[param] = input_parameters[param][pixel_id]
if i > 0:
if mu[i - 1, j] > 5:
ac_limit[j] = min(i, ac_limit[j])
ac_limit[j] = int(ac_limit[j])
weights_fit = chi_2[:ac_limit[j], j]
weights_fit = weights_fit / ndf[:ac_limit[j], j]
options['fix_mu_xt'] = True
temp = mu_xt[:ac_limit[j], j] * weights_fit
temp = np.nansum(temp)
temp = temp / np.nansum(weights_fit)
fixed_params['mu_xt'] = temp
try:
fitter = MPEFitter(histogram=histo,
cost='MLE',
pedantic=0,
print_level=0,
throw_nan=True,
fixed_params=fixed_params,
**options)
fitter.fit(ncall=ncall)
if debug:
x_label = '[LSB]'
label = 'Pixel {}'.format(pixel_id)
fitter.draw(legend=False, x_label=x_label, label=label)
fitter.draw_init(legend=False,
x_label=x_label,
label=label)
fitter.draw_fit(legend=False,
x_label=x_label,
label=label)
plt.show()
param = fitter.parameters
param_err = fitter.errors
gain[i, j] = param['gain']
sigma_e[i, j] = param['sigma_e']
sigma_s[i, j] = param['sigma_s']
baseline[i, j] = param['baseline']
mu[i, j] = param['mu']
mu_xt[i, j] = param['mu_xt']
amplitude[i, j] = param['amplitude']
gain_error[i, j] = param_err['gain']
sigma_e_error[i, j] = param_err['sigma_e']
sigma_s_error[i, j] = param_err['sigma_s']
baseline_error[i, j] = param_err['baseline']
mu_error[i, j] = param_err['mu']
mu_xt_error[i, j] = param_err['mu_xt']
amplitude_error[i, j] = param_err['amplitude']
chi_2[i, j] = fitter.fit_test() * fitter.ndf
ndf[i, j] = fitter.ndf
except Exception as e:
print(e)
print('Could not fit pixel {} for DAC level {}'.format(
pixel_id, ac_level))
np.savez(
results_filename,
gain=gain,
sigma_e=sigma_e,
sigma_s=sigma_s,
baseline=baseline,
mu=mu,
mu_xt=mu_xt,
gain_error=gain_error,
sigma_e_error=sigma_e_error,
sigma_s_error=sigma_s_error,
baseline_error=baseline_error,
mu_error=mu_error,
mu_xt_error=mu_xt_error,
chi_2=chi_2,
ndf=ndf,
pixel_ids=pixel_ids,
ac_levels=ac_levels,
amplitude=amplitude,
amplitude_error=amplitude_error,
mean=mean,
std=std,
)
if args['--save_figures']:
pass
if args['--display']:
charge_histo = Histogram1D.load(charge_histo_filename)
charge_histo.draw(index=(0, 0), log=False, legend=False)
pass
return
def get_burst(files,
plot_baseline="show",
n_previous_events=100,
threshold_lsb=2.,
output=None,
expand=10,
merge_sec=5.,
video_prefix=None,
disable_bar=False):
# get events info
events = calibration_event_stream(files, disable_bar=disable_bar)
events = fill_digicam_baseline(events)
events = tag_burst_from_moving_average_baseline(
events,
n_previous_events=n_previous_events,
threshold_lsb=threshold_lsb)
n_event = 0
timestamps = []
event_ids = []
are_burst = []
baselines = []
for event in events:
n_event += 1
timestamps.append(event.data.local_time)
event_ids.append(event.event_id)
are_burst.append(event.data.burst)
baselines.append(np.mean(event.data.digicam_baseline))
timestamps = np.array(timestamps)
event_ids = np.array(event_ids)
are_burst = np.array(are_burst)
baselines = np.array(baselines)
# plot history of the baselines
if plot_baseline is not None:
fig1 = plt.figure(figsize=(8, 6))
ax = plt.gca()
plt.xticks(rotation=70)
plt.plot_date(to_datetime(timestamps), baselines, '.')
plt.ylabel('mean baseline [LSB]')
ax.xaxis.set_major_formatter(DateFormatter('%H:%M'))
plt.tight_layout()
if plot_baseline.lower() == "show":
plt.show()
else:
plt.savefig(plot_baseline)
plt.close(fig1)
# identify the bursts
if np.all(~are_burst):
raise SystemExit('no burst detected')
are_burst = expand_mask(are_burst, iters=expand)
previous_is_burst = False
bursts = []
for event in range(n_event):
if are_burst[event]:
if not previous_is_burst:
bursts.append([event])
if event == n_event - 1 or (not are_burst[event + 1]):
bursts[-1].append(event)
previous_is_burst = True
else:
previous_is_burst = False
if np.all(~are_burst):
raise SystemExit('no burst identified')
# merge bursts which are closer than merge_sec seconds
last_burst_begin = bursts[0][0]
last_burst_end = bursts[0][1]
merged_bursts = []
for burst_idxs in bursts[1:]:
begin_idx, end_idx = burst_idxs
interval = (timestamps[begin_idx] - timestamps[last_burst_end])
if interval < merge_sec * 1e9:
last_burst_end = end_idx
else:
merged_bursts.append([last_burst_begin, last_burst_end])
last_burst_begin = begin_idx
last_burst_end = end_idx
if len(merged_bursts) == 0 or merged_bursts[-1][0] != last_burst_begin:
merged_bursts.append([last_burst_begin, last_burst_end])
bursts = merged_bursts
# output result
if output is None:
run_file = sys.stdout
else:
run_file = open(output, 'w')
run_file.write("#burst ts_start ts_end id_start id_end\n") # write header
date_format = '%Y-%m-%dT%H:%M:%S'
for i, burst_idxs in enumerate(bursts):
begin_idx, end_idx = burst_idxs
ts_begin = pd.to_datetime(timestamps[begin_idx]).strftime(date_format)
ts_end = pd.to_datetime(timestamps[end_idx]).strftime(date_format)
run_file.write(str(i) + " " + ts_begin + " " + ts_end)
run_file.write(" " + str(event_ids[begin_idx]) + " ")
run_file.write(str(event_ids[end_idx]) + "\n")
if output is not None:
run_file.close()
if video_prefix is not None:
for i, burst_idxs in enumerate(bursts):
begin_idx, end_idx = burst_idxs
events = calibration_event_stream(files, disable_bar=disable_bar)
events = fill_digicam_baseline(events)
if video_prefix != "show":
video = video_prefix + "_" + str(i) + ".mp4"
else:
video = "show"
animate_baseline(events,
video,
event_id_min=event_ids[begin_idx],
event_id_max=event_ids[end_idx])
def analyse_acdc_level(files,
max_events=None,
delay_step_ns=0.1,
delay_range_ns=(-4., 4.),
time_range_ns=(-9., 39.),
sampling_ns=4.,
normalize_range=(-3, 4),
parameters=parameters_default,
template=template_default,
adc_noise=1):
with open(parameters) as parameters_file:
calibration_parameters = yaml.load(parameters_file)
gain_pixels = np.array(calibration_parameters['gain'])
normalize_slice = np.arange(normalize_range[0],
normalize_range[1] + 1,
dtype=int)
sample_template = np.arange(time_range_ns[0], time_range_ns[1],
sampling_ns)
n_sample_template = len(sample_template)
template = NormalizedPulseTemplate.load(template)
delays = np.arange(delay_range_ns[0], delay_range_ns[1], delay_step_ns)
n_delays = len(delays)
templates_ampl = np.zeros([n_delays, n_sample_template])
templates_std = np.zeros([n_delays, n_sample_template])
index_max_template = np.zeros(n_delays, dtype=int)
for i, delay in enumerate(delays):
ampl_templ = template(sample_template + delay)
std_templ = template.std(sample_template + delay)
index_max_template[i] = np.argmax(ampl_templ)
range_integ = index_max_template[i] + normalize_slice
norm_templ = np.sum(ampl_templ[range_integ])
templates_ampl[i, :] = ampl_templ / norm_templ
templates_std[i, :] = std_templ / norm_templ
max_ampl_one_pe = np.max(templates_ampl)
events = calibration_event_stream(files,
max_events=max_events,
disable_bar=True)
events = fill_digicam_baseline(events)
events = subtract_baseline(events)
rows_norm = np.tile(
np.arange(1296, dtype=int)[:, None], [1, len(normalize_slice)])
samples_events = None
t_fit = []
charge = []
for event in events:
if samples_events is None:
n_sample = event.data.adc_samples.shape[1]
samples_events = np.arange(n_sample) * sampling_ns
adc_samples = event.data.adc_samples
idx_sample_max = np.argmax(adc_samples, axis=1)
column_norm = normalize_slice[None, :] + idx_sample_max[:, None]
# we skip pixels with max too close to the limit of the sampling window
# to be sure to be able to integrate and get normalization
good_pix = np.logical_and(np.all(column_norm < n_sample, axis=1),
np.all(column_norm >= 0, axis=1))
# we skip pixels with max too close to the limit of the sampling window
# to be sure to be able to compare with full template
mean_index_max_template = int(np.round(np.mean(index_max_template)))
index_template_rel = idx_sample_max - mean_index_max_template
good_pix = np.logical_and(good_pix, index_template_rel >= 0)
good_pix = np.logical_and(
good_pix, index_template_rel + n_sample_template - 1 < n_sample)
# we discard pixels with less that few pe
good_pix = np.logical_and(
good_pix,
np.max(adc_samples, axis=1) > 3.5 / max_ampl_one_pe)
# discard pixels with max pulse not around the right position
good_pix = np.logical_and(good_pix, idx_sample_max >= 15)
good_pix = np.logical_and(good_pix, idx_sample_max <= 16)
sample_norm = adc_samples[rows_norm[good_pix, :],
column_norm[good_pix, :]]
norm_pixels = np.sum(sample_norm, axis=1)
# discard pixels where charge is <= 2.5 LSB (0.5 pe), as normalization
# is then meaningless
norm_all = np.zeros(1296)
norm_all[good_pix] = norm_pixels
good_pix = np.logical_and(good_pix, norm_all > 2.5)
norm_pixels = norm_pixels[norm_pixels > 2.5]
charge_good_pix = norm_pixels / gain_pixels[good_pix]
adc_samples_norm = adc_samples[good_pix, :] / norm_pixels[:, None]
n_good_pix = int(np.sum(good_pix))
samples = np.arange(n_sample_template, dtype=int)[None, None, :]
column_chi2 = index_template_rel[good_pix, None, None] + samples
row_chi2 = np.tile(
np.arange(n_good_pix)[:, None, None],
[1, n_delays, n_sample_template])
adc_samples_compared = adc_samples_norm[row_chi2, column_chi2]
residual = adc_samples_compared - templates_ampl[None, :, :]
error_squared = templates_std[None, :, :] ** 2 \
+ (adc_noise/norm_pixels[:, None, None]) ** 2
chi2 = np.sum(residual**2 / error_squared, axis=2) \
/ (n_sample_template - 1)
t_fit_all = np.ones(1296) * np.nan
# estimate offset from min chi2
idx_delay_min = np.argmin(chi2, axis=1)
delays_min = delays[idx_delay_min]
delays_min[chi2[np.arange(n_good_pix), idx_delay_min] > 20] = np.nan
t_fit_all[good_pix] = delays_min
t_fit.append(-t_fit_all + idx_sample_max * sampling_ns)
charge_all = np.ones(1296) * np.nan
charge_all[good_pix] = charge_good_pix
charge.append(charge_all)
t_fit = np.array(t_fit)
charge = np.array(charge)
return charge, t_fit
def compute(files, ac_levels, dc_levels, output_filename, max_events, pixels,
integral_width, timing, saturation_threshold, pulse_tail, debug):
n_pixels = len(pixels)
n_files = len(files)
n_ac_level = len(ac_levels)
n_dc_level = len(dc_levels)
assert n_files == (n_ac_level * n_dc_level)
for file in files:
assert os.path.exists(file)
assert not os.path.exists(output_filename)
shape = (len(dc_levels), len(ac_levels), n_pixels)
amplitude_mean = np.zeros(shape)
amplitude_std = np.zeros(shape)
baseline_mean = np.zeros(shape)
baseline_std = np.zeros(shape)
charge_mean = np.zeros(shape)
charge_std = np.zeros(shape)
for i, dc_level, in tqdm(enumerate(dc_levels), total=n_dc_level):
for j, ac_level in tqdm(enumerate(ac_levels), total=n_ac_level):
index_file = i * n_ac_level + j
file = files[index_file]
events = calibration_event_stream(file, max_events=max_events)
events = fill_digicam_baseline(events)
events = subtract_baseline(events)
# events = compute_charge_with_saturation(events, integral_width=7)
events = compute_charge_with_saturation_and_threshold(
events,
integral_width=integral_width,
debug=debug,
trigger_bin=timing,
saturation_threshold=saturation_threshold,
pulse_tail=pulse_tail)
# events = compute_maximal_charge(events)
for n, event in enumerate(events):
charge_mean[i, j] += event.data.reconstructed_charge
amplitude_mean[i, j] += event.data.reconstructed_amplitude
charge_std[i, j] += event.data.reconstructed_charge**2
amplitude_std[i, j] += event.data.reconstructed_amplitude**2
baseline_mean[i, j] += event.data.baseline
baseline_std[i, j] += event.data.baseline**2
charge_mean[i, j] = charge_mean[i, j] / (n + 1)
charge_std[i, j] = charge_std[i, j] / (n + 1)
charge_std[i, j] = np.sqrt(charge_std[i, j] - charge_mean[i, j]**2)
amplitude_mean[i, j] = amplitude_mean[i, j] / (n + 1)
amplitude_std[i, j] = amplitude_std[i, j] / (n + 1)
amplitude_std[i, j] = np.sqrt(amplitude_std[i, j] -
amplitude_mean[i, j]**2)
baseline_mean[i, j] = baseline_mean[i, j] / (n + 1)
baseline_std[i, j] = baseline_std[i, j] / (n + 1)
baseline_std[i, j] = np.sqrt(baseline_std[i, j] -
baseline_mean[i, j]**2)
np.savez(output_filename,
charge_mean=charge_mean,
charge_std=charge_std,
amplitude_mean=amplitude_mean,
amplitude_std=amplitude_std,
ac_levels=ac_levels,
dc_levels=dc_levels,
baseline_mean=baseline_mean,
baseline_std=baseline_std)
def main_pipeline(files,
aux_basepath,
max_events,
dark_filename,
integral_width,
debug,
hillas_filename,
parameters_filename,
picture_threshold,
boundary_threshold,
template_filename,
saturation_threshold,
threshold_pulse,
bad_pixels=None,
disable_bar=False):
# get configuration
with open(parameters_filename) as file:
calibration_parameters = yaml.load(file)
if bad_pixels is None:
bad_pixels = get_bad_pixels(calib_file=parameters_filename,
dark_histo=dark_filename,
plot=None)
pulse_template = NormalizedPulseTemplate.load(template_filename)
pulse_area = pulse_template.integral() * u.ns
ratio = pulse_template.compute_charge_amplitude_ratio(
integral_width=integral_width, dt_sampling=4) # ~ 0.24
gain = np.array(calibration_parameters['gain']) # ~ 20 LSB / p.e.
gain_amplitude = gain * ratio
crosstalk = np.array(calibration_parameters['mu_xt'])
bias_resistance = 10 * 1E3 * u.Ohm # 10 kOhm
cell_capacitance = 50 * 1E-15 * u.Farad # 50 fF
geom = DigiCam.geometry
dark_histo = Histogram1D.load(dark_filename)
dark_baseline = dark_histo.mean()
# define pipeline
events = calibration_event_stream(files,
max_events=max_events,
disable_bar=disable_bar)
events = add_slow_data_calibration(
events,
basepath=aux_basepath,
aux_services=('DriveSystem', 'DigicamSlowControl', 'MasterSST1M',
'SafetyPLC', 'PDPSlowControl'))
events = baseline.fill_dark_baseline(events, dark_baseline)
events = baseline.fill_digicam_baseline(events)
events = tagging.tag_burst_from_moving_average_baseline(events)
events = baseline.compute_baseline_shift(events)
events = baseline.subtract_baseline(events)
events = filters.filter_clocked_trigger(events)
events = baseline.compute_nsb_rate(events, gain_amplitude, pulse_area,
crosstalk, bias_resistance,
cell_capacitance)
events = baseline.compute_gain_drop(events, bias_resistance,
cell_capacitance)
events = peak.find_pulse_with_max(events)
events = charge.compute_dynamic_charge(
events,
integral_width=integral_width,
saturation_threshold=saturation_threshold,
threshold_pulse=threshold_pulse,
debug=debug,
pulse_tail=False,
)
events = charge.compute_photo_electron(events, gains=gain)
events = charge.interpolate_bad_pixels(events, geom, bad_pixels)
events = cleaning.compute_tailcuts_clean(
events,
geom=geom,
overwrite=True,
picture_thresh=picture_threshold,
boundary_thresh=boundary_threshold,
keep_isolated_pixels=False)
events = cleaning.compute_boarder_cleaning(events, geom,
boundary_threshold)
events = cleaning.compute_dilate(events, geom)
events = image.compute_hillas_parameters(events, geom)
events = charge.compute_sample_photo_electron(events, gain_amplitude)
events = cleaning.compute_3d_cleaning(events,
geom,
n_sample=50,
threshold_sample_pe=20,
threshold_time=2.1 * u.ns,
threshold_size=0.005 * u.mm)
# create pipeline output file
output_file = Serializer(hillas_filename, mode='w', format='fits')
data_to_store = PipelineOutputContainer()
for event in events:
if debug:
print(event.hillas)
print(event.data.nsb_rate)
print(event.data.gain_drop)
print(event.data.baseline_shift)
print(event.data.border)
plot_array_camera(np.max(event.data.adc_samples, axis=-1))
plot_array_camera(
np.nanmax(event.data.reconstructed_charge, axis=-1))
plot_array_camera(event.data.cleaning_mask.astype(float))
plot_array_camera(event.data.reconstructed_number_of_pe)
plt.show()
# fill container
data_to_store.local_time = event.data.local_time
data_to_store.event_type = event.event_type
data_to_store.event_id = event.event_id
data_to_store.az = event.slow_data.DriveSystem.current_position_az
data_to_store.el = event.slow_data.DriveSystem.current_position_el
r = event.hillas.r
phi = event.hillas.phi
psi = event.hillas.psi
alpha = compute_alpha(phi.value, psi.value) * psi.unit
data_to_store.alpha = alpha
data_to_store.miss = compute_miss(r=r.value, alpha=alpha.value)
data_to_store.miss = data_to_store.miss * r.unit
data_to_store.baseline = np.mean(event.data.digicam_baseline)
data_to_store.nsb_rate = np.mean(event.data.nsb_rate)
temp_crate1 = event.slow_data.DigicamSlowControl.Crate1_T
temp_crate2 = event.slow_data.DigicamSlowControl.Crate2_T
temp_crate3 = event.slow_data.DigicamSlowControl.Crate3_T
temp_digicam = np.array(
np.hstack([temp_crate1, temp_crate2, temp_crate3]))
temp_digicam_mean = np.mean(temp_digicam[np.logical_and(
temp_digicam > 0, temp_digicam < 60)])
data_to_store.digicam_temperature = temp_digicam_mean
temp_sector1 = event.slow_data.PDPSlowControl.Sector1_T
temp_sector2 = event.slow_data.PDPSlowControl.Sector2_T
temp_sector3 = event.slow_data.PDPSlowControl.Sector3_T
temp_pdp = np.array(
np.hstack([temp_sector1, temp_sector2, temp_sector3]))
temp_pdp_mean = np.mean(temp_pdp[np.logical_and(
temp_pdp > 0, temp_pdp < 60)])
data_to_store.pdp_temperature = temp_pdp_mean
target_radec = event.slow_data.MasterSST1M.target_radec
data_to_store.target_ra = target_radec[0]
data_to_store.target_dec = target_radec[1]
status_leds = event.slow_data.SafetyPLC.SPLC_CAM_Status
# bit 8 of status_LEDs is about on/off, bit 9 about blinking
data_to_store.pointing_leds_on = bool((status_leds & 1 << 8) >> 8)
data_to_store.pointing_leds_blink = bool((status_leds & 1 << 9) >> 9)
hv_sector1 = event.slow_data.PDPSlowControl.Sector1_HV
hv_sector2 = event.slow_data.PDPSlowControl.Sector2_HV
hv_sector3 = event.slow_data.PDPSlowControl.Sector3_HV
hv_pdp = np.array(np.hstack([hv_sector1, hv_sector2, hv_sector3]),
dtype=bool)
data_to_store.all_hv_on = np.all(hv_pdp)
ghv_sector1 = event.slow_data.PDPSlowControl.Sector1_GHV
ghv_sector2 = event.slow_data.PDPSlowControl.Sector2_GHV
ghv_sector3 = event.slow_data.PDPSlowControl.Sector3_GHV
ghv_pdp = np.array(np.hstack([ghv_sector1, ghv_sector2, ghv_sector3]),
dtype=bool)
data_to_store.all_ghv_on = np.all(ghv_pdp)
is_on_source = bool(event.slow_data.DriveSystem.is_on_source)
data_to_store.is_on_source = is_on_source
is_tracking = bool(event.slow_data.DriveSystem.is_tracking)
data_to_store.is_tracking = is_tracking
data_to_store.shower = bool(event.data.shower)
data_to_store.border = bool(event.data.border)
data_to_store.burst = bool(event.data.burst)
data_to_store.saturated = bool(event.data.saturated)
for key, val in event.hillas.items():
data_to_store[key] = val
output_file.add_container(data_to_store)
try:
output_file.close()
print(hillas_filename, 'created.')
except ValueError:
print('WARNING: no data to save,', hillas_filename, 'not created.')
def test_event_type_enum_behavior():
for event in calibration_event_stream(example_file_path):
assert event.event_type in [
event.event_type.PATCH7, event.event_type.INTERNAL
]
