def test_pulse_template_from_datafiles():
files = [data_filename1, data_filename2, data_filename3]
template = NormalizedPulseTemplate.create_from_datafiles(files)
template_load = NormalizedPulseTemplate.load(template_filename_2)
time = np.linspace(-10, 30, num=101)
std = template_load.std(time)
assert np.all(np.abs(template(time) - template_load(time)) < 5 * std)
def plot_pulse_templates(pulse_shape_files,
xscale='linear',
yscale='linear',
axes=None,
pixels=None,
**kwargs):
if axes is None:
fig, axes = plt.subplots(1, 1)
else:
fig = axes.get_figure()
colors = plt.cm.rainbow(np.linspace(0, 1, len(pulse_shape_files)))
for pulse_shape_file, color in zip(pulse_shape_files, colors):
if pulse_shape_file.endswith(".fits") or \
pulse_shape_file.endswith(".fits.gz"):
template = NormalizedPulseTemplate.create_from_datafiles(
input_files=[pulse_shape_file],
min_entries_ratio=0.1,
pixels=pixels,
)
elif pulse_shape_file.endswith(".txt")or \
pulse_shape_file.endswith(".dat"):
template = NormalizedPulseTemplate.load(pulse_shape_file)
template.plot_interpolation(axes=axes,
color=color,
sigma=1,
label=pulse_shape_file,
**kwargs)
del template
axes.set_xscale(xscale, nonposx='clip')
axes.set_yscale(yscale, nonposy='clip')
axes.set_xlabel('time w.r.t half-height [ns]')
axes.set_ylabel('normalized amplitude')
return axes
def test_pulse_template_save():
template_load = NormalizedPulseTemplate.load(template_filename_2)
with tempfile.TemporaryDirectory() as tmpdirname:
template_saved_filename = os.path.join(tmpdirname, 'test.txt')
template_load.save(template_saved_filename)
template_saved = NormalizedPulseTemplate.load(template_saved_filename)
assert np.all(template_saved.time == template_load.time)
assert np.all(template_saved.amplitude == template_load.amplitude)
assert np.all(
template_saved.amplitude_std == template_load.amplitude_std)
def test_pulse_template_from_datafile():
template1 = NormalizedPulseTemplate.create_from_datafile(data_filename1)
template2 = NormalizedPulseTemplate.create_from_datafile(data_filename2)
template3 = NormalizedPulseTemplate.create_from_datafile(data_filename3)
template_load = NormalizedPulseTemplate.load(template_filename_2)
time = np.linspace(-10, 30, num=101)
std = template_load.std(time)
assert np.all(np.abs(template1(time) - template_load(time)) < 5 * std)
assert np.all(np.abs(template2(time) - template_load(time)) < 5 * std)
assert np.all(np.abs(template3(time) - template_load(time)) < 5 * std)
def test_pulse_template_ndarray_amplitude():
n_samples = 51
time = np.linspace(0, 50, num=n_samples)
amplitude = [np.ones(n_samples), np.ones(n_samples)]
amplitude = np.array(amplitude)
template = NormalizedPulseTemplate(amplitude, time)
y = template(time, amplitude=2)
assert (template.integral() == np.array([50, 50])).all()
np.testing.assert_almost_equal(y, amplitude * 2)
def test_pulse_template_normalization():
template = NormalizedPulseTemplate.load(template_filename)
assert np.max(template.amplitude) == 1
t, x = np.loadtxt(template_filename).T
# Trying with negative template
template = NormalizedPulseTemplate(-x, t)
assert np.max(template.amplitude) == 1
# Trying with non normalized template
template = NormalizedPulseTemplate(x * 0.1, t)
assert np.max(template.amplitude) == 1
def find_pulse_correlate(
events,
threshold,
pulse_template=NormalizedPulseTemplate.load(TEMPLATE_FILENAME)):
time = np.linspace(0, 91 * 4, num=91)
template = pulse_template(time, t_0=0, amplitude=1, baseline=0)
template[template < 0.1] = 0
template = np.tile(template, (1296, 1))
template = template / np.sum(template, axis=-1)[..., np.newaxis]
for count, event in enumerate(events):
adc_samples = event.data.adc_samples
pulse_mask = np.zeros(adc_samples.shape, dtype=np.bool)
mean = np.mean(adc_samples, axis=-1)
std = np.std(adc_samples, axis=-1)
adc_samples = (adc_samples - mean[..., np.newaxis])
adc_samples = adc_samples / std[..., np.newaxis]
c = correlate(adc_samples, template)
pulse_mask[:, 1:-1] = ((c[:, :-2] <= c[:, 1:-1]) &
(c[:, 1:-1] >= c[:, 2:]) &
(c[:, 1:-1] > threshold))
event.data.pulse_mask = pulse_mask
yield event
def test_pulse_template_object_get_sub_template():
n_samples = 51
time = np.linspace(0, 50, num=n_samples)
amplitude = [np.ones(n_samples), np.arange(n_samples)]
amplitude = np.array(amplitude)
template = NormalizedPulseTemplate(amplitude, time)
assert template[0].integral() == 50
np.testing.assert_almost_equal(template[1].integral(), 50 / 2)
def main(input_files,
output=None,
plot="show",
plot_separated=None,
pixels=None,
xscale="linear",
yscale="linear"):
if output is not None or plot is not None:
template = NormalizedPulseTemplate.create_from_datafiles(
input_files=input_files, min_entries_ratio=0.1, pixels=pixels)
if output is not None:
if os.path.exists(output):
os.remove(output)
template.save(output)
print(output, 'created')
if plot is not None:
fig, ax = plt.subplots(1, 1)
template.plot(axes=ax)
ax.set_xscale(xscale)
ax.set_yscale(yscale)
if plot.lower() == "show":
plt.show()
else:
plt.savefig(plot)
print(plot, 'created')
plt.close(fig)
if plot_separated is not None:
fig, ax = plt.subplots(1, 1)
plot_pulse_templates(input_files,
xscale=xscale,
yscale=yscale,
axes=ax)
if plot_separated.lower() == "show":
plt.show()
else:
plt.savefig(plot_separated)
print(plot_separated, 'created')
plt.close(fig)
def main_pipeline(files,
aux_basepath,
max_events,
dark_filename,
integral_width,
debug,
hillas_filename,
parameters_filename,
picture_threshold,
boundary_threshold,
template_filename,
saturation_threshold,
threshold_pulse,
bad_pixels=None,
disable_bar=False):
# get configuration
with open(parameters_filename) as file:
calibration_parameters = yaml.load(file)
if bad_pixels is None:
bad_pixels = get_bad_pixels(calib_file=parameters_filename,
dark_histo=dark_filename,
plot=None)
pulse_template = NormalizedPulseTemplate.load(template_filename)
pulse_area = pulse_template.integral() * u.ns
ratio = pulse_template.compute_charge_amplitude_ratio(
integral_width=integral_width, dt_sampling=4) # ~ 0.24
gain = np.array(calibration_parameters['gain']) # ~ 20 LSB / p.e.
gain_amplitude = gain * ratio
crosstalk = np.array(calibration_parameters['mu_xt'])
bias_resistance = 10 * 1E3 * u.Ohm # 10 kOhm
cell_capacitance = 50 * 1E-15 * u.Farad # 50 fF
geom = DigiCam.geometry
dark_histo = Histogram1D.load(dark_filename)
dark_baseline = dark_histo.mean()
# define pipeline
events = calibration_event_stream(files,
max_events=max_events,
disable_bar=disable_bar)
events = add_slow_data_calibration(
events,
basepath=aux_basepath,
aux_services=('DriveSystem', 'DigicamSlowControl', 'MasterSST1M',
'SafetyPLC', 'PDPSlowControl'))
events = baseline.fill_dark_baseline(events, dark_baseline)
events = baseline.fill_digicam_baseline(events)
events = tagging.tag_burst_from_moving_average_baseline(events)
events = baseline.compute_baseline_shift(events)
events = baseline.subtract_baseline(events)
events = filters.filter_clocked_trigger(events)
events = baseline.compute_nsb_rate(events, gain_amplitude, pulse_area,
crosstalk, bias_resistance,
cell_capacitance)
events = baseline.compute_gain_drop(events, bias_resistance,
cell_capacitance)
events = peak.find_pulse_with_max(events)
events = charge.compute_dynamic_charge(
events,
integral_width=integral_width,
saturation_threshold=saturation_threshold,
threshold_pulse=threshold_pulse,
debug=debug,
pulse_tail=False,
)
events = charge.compute_photo_electron(events, gains=gain)
events = charge.interpolate_bad_pixels(events, geom, bad_pixels)
events = cleaning.compute_tailcuts_clean(
events,
geom=geom,
overwrite=True,
picture_thresh=picture_threshold,
boundary_thresh=boundary_threshold,
keep_isolated_pixels=False)
events = cleaning.compute_boarder_cleaning(events, geom,
boundary_threshold)
events = cleaning.compute_dilate(events, geom)
events = image.compute_hillas_parameters(events, geom)
events = charge.compute_sample_photo_electron(events, gain_amplitude)
events = cleaning.compute_3d_cleaning(events,
geom,
n_sample=50,
threshold_sample_pe=20,
threshold_time=2.1 * u.ns,
threshold_size=0.005 * u.mm)
# create pipeline output file
output_file = Serializer(hillas_filename, mode='w', format='fits')
data_to_store = PipelineOutputContainer()
for event in events:
if debug:
print(event.hillas)
print(event.data.nsb_rate)
print(event.data.gain_drop)
print(event.data.baseline_shift)
print(event.data.border)
plot_array_camera(np.max(event.data.adc_samples, axis=-1))
plot_array_camera(
np.nanmax(event.data.reconstructed_charge, axis=-1))
plot_array_camera(event.data.cleaning_mask.astype(float))
plot_array_camera(event.data.reconstructed_number_of_pe)
plt.show()
# fill container
data_to_store.local_time = event.data.local_time
data_to_store.event_type = event.event_type
data_to_store.event_id = event.event_id
data_to_store.az = event.slow_data.DriveSystem.current_position_az
data_to_store.el = event.slow_data.DriveSystem.current_position_el
r = event.hillas.r
phi = event.hillas.phi
psi = event.hillas.psi
alpha = compute_alpha(phi.value, psi.value) * psi.unit
data_to_store.alpha = alpha
data_to_store.miss = compute_miss(r=r.value, alpha=alpha.value)
data_to_store.miss = data_to_store.miss * r.unit
data_to_store.baseline = np.mean(event.data.digicam_baseline)
data_to_store.nsb_rate = np.mean(event.data.nsb_rate)
temp_crate1 = event.slow_data.DigicamSlowControl.Crate1_T
temp_crate2 = event.slow_data.DigicamSlowControl.Crate2_T
temp_crate3 = event.slow_data.DigicamSlowControl.Crate3_T
temp_digicam = np.array(
np.hstack([temp_crate1, temp_crate2, temp_crate3]))
temp_digicam_mean = np.mean(temp_digicam[np.logical_and(
temp_digicam > 0, temp_digicam < 60)])
data_to_store.digicam_temperature = temp_digicam_mean
temp_sector1 = event.slow_data.PDPSlowControl.Sector1_T
temp_sector2 = event.slow_data.PDPSlowControl.Sector2_T
temp_sector3 = event.slow_data.PDPSlowControl.Sector3_T
temp_pdp = np.array(
np.hstack([temp_sector1, temp_sector2, temp_sector3]))
temp_pdp_mean = np.mean(temp_pdp[np.logical_and(
temp_pdp > 0, temp_pdp < 60)])
data_to_store.pdp_temperature = temp_pdp_mean
target_radec = event.slow_data.MasterSST1M.target_radec
data_to_store.target_ra = target_radec[0]
data_to_store.target_dec = target_radec[1]
status_leds = event.slow_data.SafetyPLC.SPLC_CAM_Status
# bit 8 of status_LEDs is about on/off, bit 9 about blinking
data_to_store.pointing_leds_on = bool((status_leds & 1 << 8) >> 8)
data_to_store.pointing_leds_blink = bool((status_leds & 1 << 9) >> 9)
hv_sector1 = event.slow_data.PDPSlowControl.Sector1_HV
hv_sector2 = event.slow_data.PDPSlowControl.Sector2_HV
hv_sector3 = event.slow_data.PDPSlowControl.Sector3_HV
hv_pdp = np.array(np.hstack([hv_sector1, hv_sector2, hv_sector3]),
dtype=bool)
data_to_store.all_hv_on = np.all(hv_pdp)
ghv_sector1 = event.slow_data.PDPSlowControl.Sector1_GHV
ghv_sector2 = event.slow_data.PDPSlowControl.Sector2_GHV
ghv_sector3 = event.slow_data.PDPSlowControl.Sector3_GHV
ghv_pdp = np.array(np.hstack([ghv_sector1, ghv_sector2, ghv_sector3]),
dtype=bool)
data_to_store.all_ghv_on = np.all(ghv_pdp)
is_on_source = bool(event.slow_data.DriveSystem.is_on_source)
data_to_store.is_on_source = is_on_source
is_tracking = bool(event.slow_data.DriveSystem.is_tracking)
data_to_store.is_tracking = is_tracking
data_to_store.shower = bool(event.data.shower)
data_to_store.border = bool(event.data.border)
data_to_store.burst = bool(event.data.burst)
data_to_store.saturated = bool(event.data.saturated)
for key, val in event.hillas.items():
data_to_store[key] = val
output_file.add_container(data_to_store)
try:
output_file.close()
print(hillas_filename, 'created.')
except ValueError:
print('WARNING: no data to save,', hillas_filename, 'not created.')
def analyse_acdc_level(files,
max_events=None,
delay_step_ns=0.1,
delay_range_ns=(-4., 4.),
time_range_ns=(-9., 39.),
sampling_ns=4.,
normalize_range=(-3, 4),
parameters=parameters_default,
template=template_default,
adc_noise=1):
with open(parameters) as parameters_file:
calibration_parameters = yaml.load(parameters_file)
gain_pixels = np.array(calibration_parameters['gain'])
normalize_slice = np.arange(normalize_range[0],
normalize_range[1] + 1,
dtype=int)
sample_template = np.arange(time_range_ns[0], time_range_ns[1],
sampling_ns)
n_sample_template = len(sample_template)
template = NormalizedPulseTemplate.load(template)
delays = np.arange(delay_range_ns[0], delay_range_ns[1], delay_step_ns)
n_delays = len(delays)
templates_ampl = np.zeros([n_delays, n_sample_template])
templates_std = np.zeros([n_delays, n_sample_template])
index_max_template = np.zeros(n_delays, dtype=int)
for i, delay in enumerate(delays):
ampl_templ = template(sample_template + delay)
std_templ = template.std(sample_template + delay)
index_max_template[i] = np.argmax(ampl_templ)
range_integ = index_max_template[i] + normalize_slice
norm_templ = np.sum(ampl_templ[range_integ])
templates_ampl[i, :] = ampl_templ / norm_templ
templates_std[i, :] = std_templ / norm_templ
max_ampl_one_pe = np.max(templates_ampl)
events = calibration_event_stream(files,
max_events=max_events,
disable_bar=True)
events = fill_digicam_baseline(events)
events = subtract_baseline(events)
rows_norm = np.tile(
np.arange(1296, dtype=int)[:, None], [1, len(normalize_slice)])
samples_events = None
t_fit = []
charge = []
for event in events:
if samples_events is None:
n_sample = event.data.adc_samples.shape[1]
samples_events = np.arange(n_sample) * sampling_ns
adc_samples = event.data.adc_samples
idx_sample_max = np.argmax(adc_samples, axis=1)
column_norm = normalize_slice[None, :] + idx_sample_max[:, None]
# we skip pixels with max too close to the limit of the sampling window
# to be sure to be able to integrate and get normalization
good_pix = np.logical_and(np.all(column_norm < n_sample, axis=1),
np.all(column_norm >= 0, axis=1))
# we skip pixels with max too close to the limit of the sampling window
# to be sure to be able to compare with full template
mean_index_max_template = int(np.round(np.mean(index_max_template)))
index_template_rel = idx_sample_max - mean_index_max_template
good_pix = np.logical_and(good_pix, index_template_rel >= 0)
good_pix = np.logical_and(
good_pix, index_template_rel + n_sample_template - 1 < n_sample)
# we discard pixels with less that few pe
good_pix = np.logical_and(
good_pix,
np.max(adc_samples, axis=1) > 3.5 / max_ampl_one_pe)
# discard pixels with max pulse not around the right position
good_pix = np.logical_and(good_pix, idx_sample_max >= 15)
good_pix = np.logical_and(good_pix, idx_sample_max <= 16)
sample_norm = adc_samples[rows_norm[good_pix, :],
column_norm[good_pix, :]]
norm_pixels = np.sum(sample_norm, axis=1)
# discard pixels where charge is <= 2.5 LSB (0.5 pe), as normalization
# is then meaningless
norm_all = np.zeros(1296)
norm_all[good_pix] = norm_pixels
good_pix = np.logical_and(good_pix, norm_all > 2.5)
norm_pixels = norm_pixels[norm_pixels > 2.5]
charge_good_pix = norm_pixels / gain_pixels[good_pix]
adc_samples_norm = adc_samples[good_pix, :] / norm_pixels[:, None]
n_good_pix = int(np.sum(good_pix))
samples = np.arange(n_sample_template, dtype=int)[None, None, :]
column_chi2 = index_template_rel[good_pix, None, None] + samples
row_chi2 = np.tile(
np.arange(n_good_pix)[:, None, None],
[1, n_delays, n_sample_template])
adc_samples_compared = adc_samples_norm[row_chi2, column_chi2]
residual = adc_samples_compared - templates_ampl[None, :, :]
error_squared = templates_std[None, :, :] ** 2 \
+ (adc_noise/norm_pixels[:, None, None]) ** 2
chi2 = np.sum(residual**2 / error_squared, axis=2) \
/ (n_sample_template - 1)
t_fit_all = np.ones(1296) * np.nan
# estimate offset from min chi2
idx_delay_min = np.argmin(chi2, axis=1)
delays_min = delays[idx_delay_min]
delays_min[chi2[np.arange(n_good_pix), idx_delay_min] > 20] = np.nan
t_fit_all[good_pix] = delays_min
t_fit.append(-t_fit_all + idx_sample_max * sampling_ns)
charge_all = np.ones(1296) * np.nan
charge_all[good_pix] = charge_good_pix
charge.append(charge_all)
t_fit = np.array(t_fit)
charge = np.array(charge)
return charge, t_fit
def data_quality(
files, dark_filename, time_step, fits_filename, load_files,
histo_filename, rate_plot_filename, baseline_plot_filename,
nsb_plot_filename, parameters_filename, template_filename,
aux_basepath, threshold_sample_pe=20.,
bias_resistance=1e4 * u.Ohm, cell_capacitance=5e-14 * u.Farad,
disable_bar=False, aux_services=('DriveSystem',)
):
input_dir = np.unique([os.path.dirname(file) for file in files])
if len(input_dir) > 1:
raise AttributeError("input files must be from the same directories")
input_dir = input_dir[0]
if aux_basepath.lower() == "search":
aux_basepath = input_dir.replace('/raw/', '/aux/')
print("auxiliary files are expected in", aux_basepath)
with open(parameters_filename) as file:
calibration_parameters = yaml.load(file)
pulse_template = NormalizedPulseTemplate.load(template_filename)
pulse_area = pulse_template.integral() * u.ns
gain_integral = np.array(calibration_parameters['gain'])
charge_to_amplitude = pulse_template.compute_charge_amplitude_ratio(7, 4)
gain_amplitude = gain_integral * charge_to_amplitude
crosstalk = np.array(calibration_parameters['mu_xt'])
pixel_id = np.arange(1296)
n_pixels = len(pixel_id)
dark_histo = Histogram1D.load(dark_filename)
dark_baseline = dark_histo.mean()
if not load_files:
events = calibration_event_stream(files, disable_bar=disable_bar)
events = add_slow_data_calibration(
events, basepath=aux_basepath, aux_services=aux_services
)
events = fill_digicam_baseline(events)
events = fill_dark_baseline(events, dark_baseline)
events = subtract_baseline(events)
events = compute_baseline_shift(events)
events = compute_nsb_rate(
events, gain_amplitude, pulse_area, crosstalk, bias_resistance,
cell_capacitance
)
events = compute_gain_drop(events, bias_resistance, cell_capacitance)
events = compute_sample_photo_electron(events, gain_amplitude)
events = tag_burst_from_moving_average_baseline(
events, n_previous_events=100, threshold_lsb=5
)
events = compute_3d_cleaning(events, geom=DigiCam.geometry,
threshold_sample_pe=threshold_sample_pe)
init_time = 0
baseline = 0
count = 0
shower_count = 0
az = 0
el = 0
container = DataQualityContainer()
file = Serializer(fits_filename, mode='w', format='fits')
baseline_histo = Histogram1D(
data_shape=(n_pixels,),
bin_edges=np.arange(4096)
)
for i, event in enumerate(events):
new_time = event.data.local_time
if init_time == 0:
init_time = new_time
count += 1
baseline += np.mean(event.data.digicam_baseline)
az += event.slow_data.DriveSystem.current_position_az
el += event.slow_data.DriveSystem.current_position_el
time_diff = new_time - init_time
if event.data.shower:
shower_count += 1
baseline_histo.fill(event.data.digicam_baseline.reshape(-1, 1))
if time_diff > time_step and i > 0:
trigger_rate = count / time_diff
shower_rate = shower_count / time_diff
baseline = baseline / count
az = az / count
el = el / count
container.trigger_rate = trigger_rate
container.baseline = baseline
container.time = (new_time + init_time) / 2
container.shower_rate = shower_rate
container.burst = event.data.burst
nsb_rate = event.data.nsb_rate
container.nsb_rate = np.nanmean(nsb_rate).value
container.current_position_az = az
container.current_position_el = el
baseline = 0
count = 0
init_time = 0
shower_count = 0
az = 0
el = 0
file.add_container(container)
output_path = os.path.dirname(histo_filename)
if not os.path.exists(output_path):
os.makedirs(output_path)
baseline_histo.save(histo_filename)
print(histo_filename, 'created.')
file.close()
print(fits_filename, 'created.')
data = Table.read(fits_filename, format='fits')
data = data.to_pandas()
data['time'] = pd.to_datetime(data['time'], utc=True)
data = data.set_index('time')
if rate_plot_filename is not None:
fig1 = plt.figure()
ax = plt.gca()
plt.xticks(rotation=70)
plt.plot(data['trigger_rate']*1E9, '.', label='trigger rate')
plt.plot(data['shower_rate']*1E9, '.', label='shower_rate')
plt.ylabel('rate [Hz]')
plt.legend({'trigger rate', 'shower rate'})
xlim = plt.xlim()
plt.xlim(xlim[0] - 1e-3, xlim[1] + 1e-3) # extra min on the sides
if rate_plot_filename == "show":
plt.show()
else:
output_path = os.path.dirname(rate_plot_filename)
if not (output_path == '' or os.path.exists(output_path)):
os.makedirs(output_path)
plt.savefig(rate_plot_filename)
plt.close(fig1)
if baseline_plot_filename is not None:
fig2 = plt.figure(figsize=(8, 6))
ax = plt.gca()
data_burst = data[data['burst']]
data_good = data[~data['burst']]
plt.xticks(rotation=70)
plt.plot(data_good['baseline'], '.', label='good', ms=2)
plt.plot(data_burst['baseline'], '.', label='burst', ms=2)
plt.ylabel('Baseline [LSB]')
xlim = plt.xlim()
plt.xlim(xlim[0] - 1e-3, xlim[1] + 1e-3) # extra min on the sides
if rate_plot_filename == "show":
plt.show()
else:
output_path = os.path.dirname(baseline_plot_filename)
if not (output_path == '' or os.path.exists(output_path)):
os.makedirs(output_path)
plt.savefig(baseline_plot_filename)
plt.close(fig2)
if nsb_plot_filename is not None:
fig3 = plt.figure()
ax = fig3.add_subplot(111)
data.plot(y='nsb_rate', ax=ax)
ax.set_ylabel('$f_{NSB}$ [GHz]')
if nsb_plot_filename == "show":
plt.show()
else:
fig3.savefig(nsb_plot_filename)
plt.close(fig3)
return
def nsb_rate(
baseline_histo_file, dark_histo_file, param_file, template_filename,
plot="show", plot_nsb_range=None, norm="log",
bias_resistance=1e4 * u.Ohm, cell_capacitance=5e-14 * u.Farad
):
baseline_histo = Histogram1D.load(baseline_histo_file)
dark_histo = Histogram1D.load(dark_histo_file)
baseline_shift = baseline_histo.mean()-dark_histo.mean()
n_pixel = len(DigiCam.geometry.neighbors)
pixels = np.arange(n_pixel, dtype=int)
with open(param_file) as file:
pulse_template = NormalizedPulseTemplate.load(template_filename)
pulse_area = pulse_template.integral() * u.ns
charge_to_amplitude = pulse_template.compute_charge_amplitude_ratio(7, 4)
calibration_parameters = yaml.load(file)
gain_integral = np.array(calibration_parameters['gain'])
gain_amplitude = gain_integral * charge_to_amplitude
crosstalk = np.array(calibration_parameters['mu_xt'])
rate = _compute_nsb_rate(
baseline_shift=baseline_shift, gain=gain_amplitude,
pulse_area=pulse_area, crosstalk=crosstalk,
bias_resistance=bias_resistance, cell_capacitance=cell_capacitance
)
bad_pixels = get_bad_pixels(
calib_file=param_file, nsigma_gain=5, nsigma_elecnoise=5,
dark_histo=dark_histo_file, nsigma_dark=8, plot=None, output=None
)
bad_pixels = np.unique(np.hstack(
(
bad_pixels,
pixels[rate < 0],
pixels[rate > 5 * u.GHz]
)
))
avg_matrix = _get_average_matrix_bad_pixels(DigiCam.geometry, bad_pixels)
good_pixels_mask = np.ones(n_pixel, dtype=bool)
good_pixels_mask[bad_pixels] = False
good_pixels = pixels[good_pixels_mask]
rate[bad_pixels] = avg_matrix[bad_pixels, :].dot(rate[good_pixels])
if plot is None:
return rate
fig1, ax = plt.subplots(1, 1)
display = CameraDisplay(DigiCam.geometry, ax=ax, norm=norm,
title='NSB rate [GHz]')
rate_ghz = rate.to(u.GHz).value
display.image = rate_ghz
if plot_nsb_range is None:
plot_nsb_range = (np.min(rate_ghz), np.max(rate_ghz))
display.set_limits_minmax(*plot_nsb_range)
display.add_colorbar(ax=ax)
display.highlight_pixels(bad_pixels, color='r', linewidth=2)
plt.tight_layout()
output_path = os.path.dirname(plot)
if plot == "show" or \
(output_path != "" and not os.path.isdir(output_path)):
if not plot == "show":
print('WARNING: Path ' + output_path + ' for output trigger ' +
'uniformity does not exist, displaying the plot instead.\n')
plt.show()
else:
plt.savefig(plot)
print(plot, 'created')
plt.close(fig1)
return rate
def nsb_rate(
files, aux_basepath, dark_histo_file, param_file, template_filename,
output=None, plot="show", plot_nsb_range=None, norm="log",
plot_baselines=False, disable_bar=False, max_events=None, n_skip=10,
stars=True,
bias_resistance=1e4 * u.Ohm, cell_capacitance=5e-14 * u.Farad
):
files = np.atleast_1d(files)
if len(files) == 1 and not files[0].endswith('.fz'):
table = Table.read(files[0])[:max_events]
data = dict(table)
data['nsb_rate'] = np.array(data['nsb_rate']) * u.GHz
else:
dark_histo = Histogram1D.load(dark_histo_file)
n_pixel = len(DigiCam.geometry.neighbors)
pixels = np.arange(n_pixel, dtype=int)
with open(param_file) as file:
pulse_template = NormalizedPulseTemplate.load(template_filename)
pulse_area = pulse_template.integral() * u.ns
charge_to_amplitude = pulse_template.compute_charge_amplitude_ratio(7, 4)
calibration_parameters = yaml.load(file)
gain_integral = np.array(calibration_parameters['gain'])
gain_amplitude = gain_integral * charge_to_amplitude
crosstalk = np.array(calibration_parameters['mu_xt'])
events = calibration_event_stream(files, max_events=max_events,
disable_bar=disable_bar)
events = add_slow_data_calibration(
events, basepath=aux_basepath,
aux_services=('DriveSystem', )
)
data = {
"baseline": [],
"nsb_rate": [],
"good_pixels_mask": [],
"timestamp": [],
"event_id": [],
"az": [],
"el": [],
}
bad_pixels = get_bad_pixels(
calib_file=param_file, nsigma_gain=5, nsigma_elecnoise=5,
dark_histo=dark_histo_file, nsigma_dark=8, plot=None, output=None
)
events_skipped = 0
for event in events:
if event.event_type.INTERNAL not in event.event_type:
continue
events_skipped += 1
if events_skipped < n_skip:
continue
events_skipped = 0
data['baseline'].append(event.data.digicam_baseline)
baseline_shift = event.data.digicam_baseline - dark_histo.mean()
rate = _compute_nsb_rate(
baseline_shift=baseline_shift, gain=gain_amplitude,
pulse_area=pulse_area, crosstalk=crosstalk,
bias_resistance=bias_resistance, cell_capacitance=cell_capacitance
)
bad_pixels_event = np.unique(np.hstack(
(
bad_pixels,
pixels[rate < 0],
pixels[rate > 5 * u.GHz]
)
))
avg_matrix = _get_average_matrix_bad_pixels(
DigiCam.geometry, bad_pixels_event
)
good_pixels_mask = np.ones(n_pixel, dtype=bool)
good_pixels_mask[bad_pixels_event] = False
good_pixels = pixels[good_pixels_mask]
rate[bad_pixels_event] = avg_matrix[bad_pixels_event, :].dot(
rate[good_pixels]
)
data['good_pixels_mask'].append(good_pixels_mask)
data['timestamp'].append(event.data.local_time)
data['event_id'].append(event.event_id)
data['nsb_rate'].append(rate)
data['az'].append(event.slow_data.DriveSystem.current_position_az)
data['el'].append(event.slow_data.DriveSystem.current_position_el)
data['nsb_rate'] = np.array(data['nsb_rate']) * u.GHz
if output is not None:
table = Table(data)
if os.path.isfile(output):
os.remove(output)
table.write(output, format='fits')
time_obs = Time(
np.array(data['timestamp'], dtype=np.float64) * 1e-9,
format='unix'
)
if plot_baselines:
fig2, (ax1, ax2) = plt.subplots(2, 1, figsize=(16, 8), dpi=50)
baseline_std = np.std(data['baseline'], axis=0)
ax1.hist(baseline_std, 100)
ax1.set_xlabel('std(baseline) [LSB]')
# pixels_shown = np.arange(len(baseline_std))[baseline_std > 10]
pixels_shown = [834,]
ax2.plot_date(
time_obs.to_datetime(),
data['baseline'][:, pixels_shown],
'-'
)
ax2.set_xlabel('time')
ax2.set_ylabel('baseline [LSB]')
plt.tight_layout()
plt.show()
plt.close(fig2)
az_obs = np.array(data['az']) * u.deg
el_obs = np.array(data['el']) * u.deg
n_event = len(data['timestamp'])
fig1, ax = plt.subplots(1, 1, figsize=(16, 12), dpi=50)
date = datetime.fromtimestamp(data['timestamp'][0]*1e-9)
date_str = date.strftime("%H:%M:%S")
display = CameraDisplay(
DigiCam.geometry, ax=ax, norm=norm,
title='NSB rate [GHz], t=' + date_str
)
rate_ghz = np.array(data['nsb_rate'][0].to(u.GHz).value)
display.image = rate_ghz
if plot_nsb_range is None:
min_range_rate = np.max([np.min(rate_ghz), 50e-3])
plot_nsb_range = (min_range_rate, np.max(rate_ghz))
display.set_limits_minmax(*plot_nsb_range)
display.add_colorbar(ax=ax)
bad_pixels = np.arange(
len(data['good_pixels_mask'][0])
)[~data['good_pixels_mask'][0]]
display.highlight_pixels(bad_pixels, color='r', linewidth=2)
display.axes.set_xlim([-500., 500.])
display.axes.set_ylim([-500., 500.])
plt.tight_layout()
if stars is True:
stars_az, stars_alt, stars_pmag = get_stars_in_fov(
az_obs[0], el_obs[0], time_obs
)
stars_x, stars_y = transform_azel_to_xy(
stars_az, stars_alt, az_obs, el_obs
)
point_stars = []
for index_star in range(len(stars_pmag)):
point_star, = ax.plot(
stars_x[index_star, 0],
stars_y[index_star, 0],
'ok',
ms=20-2*stars_pmag[index_star],
mew=3,
mfc='None'
)
point_stars.append(point_star)
def update(i, display):
print('frame', i, '/', len(data['timestamp']))
display.image = data['nsb_rate'][i].to(u.GHz).value
date = datetime.fromtimestamp(data['timestamp'][i] * 1e-9)
date_str = date.strftime("%H:%M:%S")
display.axes.set_title('NSB rate [GHz], t=' + date_str)
bad_pixels = np.arange(
len(data['good_pixels_mask'][i])
)[~data['good_pixels_mask'][i]]
display.highlight_pixels(
bad_pixels, color='r', linewidth=2
)
if stars is True:
for index_star in range(len(stars_pmag)):
point_stars[index_star].set_xdata(
stars_x[index_star, i]
)
point_stars[index_star].set_ydata(
stars_y[index_star, i]
)
anim = FuncAnimation(
fig1,
update,
frames=len(data['timestamp']),
interval=20,
fargs=(display, )
)
Writer = animation.writers['ffmpeg']
writer = Writer(fps=50, metadata=dict(artist='Y. Renier'),
bitrate=4000, codec='h263p')
output_path = os.path.dirname(plot)
if plot == "show" or \
(output_path != "" and not os.path.isdir(output_path)):
if not plot == "show":
print('WARNING: Path ' + output_path + ' for output trigger ' +
'uniformity does not exist, displaying the plot instead.\n')
display.enable_pixel_picker()
plt.show()
else:
anim.save(plot, writer=writer)
print(plot, 'created')
plt.close(fig1)
def test_charge_amplitude_ratio():
template = NormalizedPulseTemplate.load(template_filename)
ratio = template.compute_charge_amplitude_ratio(7, 4)
assert RATIO_CHARGE_AMPLITUDE == ratio
def test_pulse_template_creation_with_file():
template = NormalizedPulseTemplate.load(template_filename)
t, x = np.loadtxt(template_filename).T
assert (t == template.time).all()
import os
import matplotlib.pyplot as plt
import numpy as np
from iminuit import Minuit
from pkg_resources import resource_filename
from probfit import Chi2Regression
from scipy.ndimage.filters import convolve1d
from digicampipe.utils.pulse_template import NormalizedPulseTemplate
TEMPLATE_FILENAME = resource_filename(
'digicampipe',
os.path.join('tests', 'resources', 'pulse_SST-1M_pixel_0.dat'))
PULSE_TEMPLATE = NormalizedPulseTemplate.load(TEMPLATE_FILENAME)
def compute_charge(events, integral_width, shift):
"""
:param events: a stream of events
:param integral_width: width of the integration window
:param shift: shift to the pulse index
:return:
"""
for count, event in enumerate(events):
adc_samples = event.data.adc_samples
pulse_mask = event.data.pulse_mask
def test_pulse_template_plot():
template = NormalizedPulseTemplate.load(template_filename)
template.plot()
template.plot_interpolation()
plot_pulse_templates([template_filename], xscale='linear', yscale='linear')
plot_pulse_templates([template_filename], xscale='log', yscale='log')
def test_pulse_template_integral():
template = NormalizedPulseTemplate.load(template_filename)
assert template.integral() == PULSE_AREA
def test_pulse_template_creation_with_file_with_std():
template = NormalizedPulseTemplate.load(template_filename_2)
t, x, std = np.loadtxt(template_filename_2).T
assert np.all(t == template.time)