def test_bad_pixels_save(): with tempfile.TemporaryDirectory() as tmp_dir_name: out = os.path.join(tmp_dir_name, 'test.yml') get_bad_pixels(parameters_filename, plot=None, output=out) assert os.path.isfile(out) with open(out) as file: params = yaml.load(file) assert 'bad_pixels' in params.keys()
def test_bad_pixels_params(): bad_pixels = get_bad_pixels(calib_file=parameters_filename, nsigma_gain=5, nsigma_elecnoise=5, plot=None, output=None) assert np.all(bad_pixels == BAD_PIXELS_PARM)
def test_bad_pixels_dark(): with tempfile.TemporaryDirectory() as tmp_dir_name: dark_filename = os.path.join(tmp_dir_name, 'dark.pk') compute_raw(files=[dark200_file_path], max_events=None, pixel_id=convert_pixel_args(None), filename=dark_filename) bad_pixels = get_bad_pixels(dark_histo=dark_filename, nsigma_dark=8, plot=None, output=None) assert np.all(bad_pixels == BAD_PIXELS_DARK)
def test_bad_pixels_all(): with tempfile.TemporaryDirectory() as tmp_dir_name: dark_filename = os.path.join(tmp_dir_name, 'dark.pk') compute_raw(files=[dark200_file_path], max_events=None, pixel_id=convert_pixel_args(None), filename=dark_filename) bad_pixels = get_bad_pixels(calib_file=parameters_filename, nsigma_gain=5, nsigma_elecnoise=5, dark_histo=dark_filename, nsigma_dark=8, plot=None, output=None) bad_pixels_true = BAD_PIXELS_PARM bad_pixels_true.extend(BAD_PIXELS_DARK) bad_pixels_true = np.unique(np.sort(bad_pixels_true)) assert np.all(bad_pixels == bad_pixels_true)
def main_pipeline(files, aux_basepath, max_events, dark_filename, integral_width, debug, hillas_filename, parameters_filename, picture_threshold, boundary_threshold, template_filename, saturation_threshold, threshold_pulse, bad_pixels=None, disable_bar=False): # get configuration with open(parameters_filename) as file: calibration_parameters = yaml.load(file) if bad_pixels is None: bad_pixels = get_bad_pixels(calib_file=parameters_filename, dark_histo=dark_filename, plot=None) pulse_template = NormalizedPulseTemplate.load(template_filename) pulse_area = pulse_template.integral() * u.ns ratio = pulse_template.compute_charge_amplitude_ratio( integral_width=integral_width, dt_sampling=4) # ~ 0.24 gain = np.array(calibration_parameters['gain']) # ~ 20 LSB / p.e. gain_amplitude = gain * ratio crosstalk = np.array(calibration_parameters['mu_xt']) bias_resistance = 10 * 1E3 * u.Ohm # 10 kOhm cell_capacitance = 50 * 1E-15 * u.Farad # 50 fF geom = DigiCam.geometry dark_histo = Histogram1D.load(dark_filename) dark_baseline = dark_histo.mean() # define pipeline events = calibration_event_stream(files, max_events=max_events, disable_bar=disable_bar) events = add_slow_data_calibration( events, basepath=aux_basepath, aux_services=('DriveSystem', 'DigicamSlowControl', 'MasterSST1M', 'SafetyPLC', 'PDPSlowControl')) events = baseline.fill_dark_baseline(events, dark_baseline) events = baseline.fill_digicam_baseline(events) events = tagging.tag_burst_from_moving_average_baseline(events) events = baseline.compute_baseline_shift(events) events = baseline.subtract_baseline(events) events = filters.filter_clocked_trigger(events) events = baseline.compute_nsb_rate(events, gain_amplitude, pulse_area, crosstalk, bias_resistance, cell_capacitance) events = baseline.compute_gain_drop(events, bias_resistance, cell_capacitance) events = peak.find_pulse_with_max(events) events = charge.compute_dynamic_charge( events, integral_width=integral_width, saturation_threshold=saturation_threshold, threshold_pulse=threshold_pulse, debug=debug, pulse_tail=False, ) events = charge.compute_photo_electron(events, gains=gain) events = charge.interpolate_bad_pixels(events, geom, bad_pixels) events = cleaning.compute_tailcuts_clean( events, geom=geom, overwrite=True, picture_thresh=picture_threshold, boundary_thresh=boundary_threshold, keep_isolated_pixels=False) events = cleaning.compute_boarder_cleaning(events, geom, boundary_threshold) events = cleaning.compute_dilate(events, geom) events = image.compute_hillas_parameters(events, geom) events = charge.compute_sample_photo_electron(events, gain_amplitude) events = cleaning.compute_3d_cleaning(events, geom, n_sample=50, threshold_sample_pe=20, threshold_time=2.1 * u.ns, threshold_size=0.005 * u.mm) # create pipeline output file output_file = Serializer(hillas_filename, mode='w', format='fits') data_to_store = PipelineOutputContainer() for event in events: if debug: print(event.hillas) print(event.data.nsb_rate) print(event.data.gain_drop) print(event.data.baseline_shift) print(event.data.border) plot_array_camera(np.max(event.data.adc_samples, axis=-1)) plot_array_camera( np.nanmax(event.data.reconstructed_charge, axis=-1)) plot_array_camera(event.data.cleaning_mask.astype(float)) plot_array_camera(event.data.reconstructed_number_of_pe) plt.show() # fill container data_to_store.local_time = event.data.local_time data_to_store.event_type = event.event_type data_to_store.event_id = event.event_id data_to_store.az = event.slow_data.DriveSystem.current_position_az data_to_store.el = event.slow_data.DriveSystem.current_position_el r = event.hillas.r phi = event.hillas.phi psi = event.hillas.psi alpha = compute_alpha(phi.value, psi.value) * psi.unit data_to_store.alpha = alpha data_to_store.miss = compute_miss(r=r.value, alpha=alpha.value) data_to_store.miss = data_to_store.miss * r.unit data_to_store.baseline = np.mean(event.data.digicam_baseline) data_to_store.nsb_rate = np.mean(event.data.nsb_rate) temp_crate1 = event.slow_data.DigicamSlowControl.Crate1_T temp_crate2 = event.slow_data.DigicamSlowControl.Crate2_T temp_crate3 = event.slow_data.DigicamSlowControl.Crate3_T temp_digicam = np.array( np.hstack([temp_crate1, temp_crate2, temp_crate3])) temp_digicam_mean = np.mean(temp_digicam[np.logical_and( temp_digicam > 0, temp_digicam < 60)]) data_to_store.digicam_temperature = temp_digicam_mean temp_sector1 = event.slow_data.PDPSlowControl.Sector1_T temp_sector2 = event.slow_data.PDPSlowControl.Sector2_T temp_sector3 = event.slow_data.PDPSlowControl.Sector3_T temp_pdp = np.array( np.hstack([temp_sector1, temp_sector2, temp_sector3])) temp_pdp_mean = np.mean(temp_pdp[np.logical_and( temp_pdp > 0, temp_pdp < 60)]) data_to_store.pdp_temperature = temp_pdp_mean target_radec = event.slow_data.MasterSST1M.target_radec data_to_store.target_ra = target_radec[0] data_to_store.target_dec = target_radec[1] status_leds = event.slow_data.SafetyPLC.SPLC_CAM_Status # bit 8 of status_LEDs is about on/off, bit 9 about blinking data_to_store.pointing_leds_on = bool((status_leds & 1 << 8) >> 8) data_to_store.pointing_leds_blink = bool((status_leds & 1 << 9) >> 9) hv_sector1 = event.slow_data.PDPSlowControl.Sector1_HV hv_sector2 = event.slow_data.PDPSlowControl.Sector2_HV hv_sector3 = event.slow_data.PDPSlowControl.Sector3_HV hv_pdp = np.array(np.hstack([hv_sector1, hv_sector2, hv_sector3]), dtype=bool) data_to_store.all_hv_on = np.all(hv_pdp) ghv_sector1 = event.slow_data.PDPSlowControl.Sector1_GHV ghv_sector2 = event.slow_data.PDPSlowControl.Sector2_GHV ghv_sector3 = event.slow_data.PDPSlowControl.Sector3_GHV ghv_pdp = np.array(np.hstack([ghv_sector1, ghv_sector2, ghv_sector3]), dtype=bool) data_to_store.all_ghv_on = np.all(ghv_pdp) is_on_source = bool(event.slow_data.DriveSystem.is_on_source) data_to_store.is_on_source = is_on_source is_tracking = bool(event.slow_data.DriveSystem.is_tracking) data_to_store.is_tracking = is_tracking data_to_store.shower = bool(event.data.shower) data_to_store.border = bool(event.data.border) data_to_store.burst = bool(event.data.burst) data_to_store.saturated = bool(event.data.saturated) for key, val in event.hillas.items(): data_to_store[key] = val output_file.add_container(data_to_store) try: output_file.close() print(hillas_filename, 'created.') except ValueError: print('WARNING: no data to save,', hillas_filename, 'not created.')
def nsb_rate( baseline_histo_file, dark_histo_file, param_file, template_filename, plot="show", plot_nsb_range=None, norm="log", bias_resistance=1e4 * u.Ohm, cell_capacitance=5e-14 * u.Farad ): baseline_histo = Histogram1D.load(baseline_histo_file) dark_histo = Histogram1D.load(dark_histo_file) baseline_shift = baseline_histo.mean()-dark_histo.mean() n_pixel = len(DigiCam.geometry.neighbors) pixels = np.arange(n_pixel, dtype=int) with open(param_file) as file: pulse_template = NormalizedPulseTemplate.load(template_filename) pulse_area = pulse_template.integral() * u.ns charge_to_amplitude = pulse_template.compute_charge_amplitude_ratio(7, 4) calibration_parameters = yaml.load(file) gain_integral = np.array(calibration_parameters['gain']) gain_amplitude = gain_integral * charge_to_amplitude crosstalk = np.array(calibration_parameters['mu_xt']) rate = _compute_nsb_rate( baseline_shift=baseline_shift, gain=gain_amplitude, pulse_area=pulse_area, crosstalk=crosstalk, bias_resistance=bias_resistance, cell_capacitance=cell_capacitance ) bad_pixels = get_bad_pixels( calib_file=param_file, nsigma_gain=5, nsigma_elecnoise=5, dark_histo=dark_histo_file, nsigma_dark=8, plot=None, output=None ) bad_pixels = np.unique(np.hstack( ( bad_pixels, pixels[rate < 0], pixels[rate > 5 * u.GHz] ) )) avg_matrix = _get_average_matrix_bad_pixels(DigiCam.geometry, bad_pixels) good_pixels_mask = np.ones(n_pixel, dtype=bool) good_pixels_mask[bad_pixels] = False good_pixels = pixels[good_pixels_mask] rate[bad_pixels] = avg_matrix[bad_pixels, :].dot(rate[good_pixels]) if plot is None: return rate fig1, ax = plt.subplots(1, 1) display = CameraDisplay(DigiCam.geometry, ax=ax, norm=norm, title='NSB rate [GHz]') rate_ghz = rate.to(u.GHz).value display.image = rate_ghz if plot_nsb_range is None: plot_nsb_range = (np.min(rate_ghz), np.max(rate_ghz)) display.set_limits_minmax(*plot_nsb_range) display.add_colorbar(ax=ax) display.highlight_pixels(bad_pixels, color='r', linewidth=2) plt.tight_layout() output_path = os.path.dirname(plot) if plot == "show" or \ (output_path != "" and not os.path.isdir(output_path)): if not plot == "show": print('WARNING: Path ' + output_path + ' for output trigger ' + 'uniformity does not exist, displaying the plot instead.\n') plt.show() else: plt.savefig(plot) print(plot, 'created') plt.close(fig1) return rate
def nsb_rate( files, aux_basepath, dark_histo_file, param_file, template_filename, output=None, plot="show", plot_nsb_range=None, norm="log", plot_baselines=False, disable_bar=False, max_events=None, n_skip=10, stars=True, bias_resistance=1e4 * u.Ohm, cell_capacitance=5e-14 * u.Farad ): files = np.atleast_1d(files) if len(files) == 1 and not files[0].endswith('.fz'): table = Table.read(files[0])[:max_events] data = dict(table) data['nsb_rate'] = np.array(data['nsb_rate']) * u.GHz else: dark_histo = Histogram1D.load(dark_histo_file) n_pixel = len(DigiCam.geometry.neighbors) pixels = np.arange(n_pixel, dtype=int) with open(param_file) as file: pulse_template = NormalizedPulseTemplate.load(template_filename) pulse_area = pulse_template.integral() * u.ns charge_to_amplitude = pulse_template.compute_charge_amplitude_ratio(7, 4) calibration_parameters = yaml.load(file) gain_integral = np.array(calibration_parameters['gain']) gain_amplitude = gain_integral * charge_to_amplitude crosstalk = np.array(calibration_parameters['mu_xt']) events = calibration_event_stream(files, max_events=max_events, disable_bar=disable_bar) events = add_slow_data_calibration( events, basepath=aux_basepath, aux_services=('DriveSystem', ) ) data = { "baseline": [], "nsb_rate": [], "good_pixels_mask": [], "timestamp": [], "event_id": [], "az": [], "el": [], } bad_pixels = get_bad_pixels( calib_file=param_file, nsigma_gain=5, nsigma_elecnoise=5, dark_histo=dark_histo_file, nsigma_dark=8, plot=None, output=None ) events_skipped = 0 for event in events: if event.event_type.INTERNAL not in event.event_type: continue events_skipped += 1 if events_skipped < n_skip: continue events_skipped = 0 data['baseline'].append(event.data.digicam_baseline) baseline_shift = event.data.digicam_baseline - dark_histo.mean() rate = _compute_nsb_rate( baseline_shift=baseline_shift, gain=gain_amplitude, pulse_area=pulse_area, crosstalk=crosstalk, bias_resistance=bias_resistance, cell_capacitance=cell_capacitance ) bad_pixels_event = np.unique(np.hstack( ( bad_pixels, pixels[rate < 0], pixels[rate > 5 * u.GHz] ) )) avg_matrix = _get_average_matrix_bad_pixels( DigiCam.geometry, bad_pixels_event ) good_pixels_mask = np.ones(n_pixel, dtype=bool) good_pixels_mask[bad_pixels_event] = False good_pixels = pixels[good_pixels_mask] rate[bad_pixels_event] = avg_matrix[bad_pixels_event, :].dot( rate[good_pixels] ) data['good_pixels_mask'].append(good_pixels_mask) data['timestamp'].append(event.data.local_time) data['event_id'].append(event.event_id) data['nsb_rate'].append(rate) data['az'].append(event.slow_data.DriveSystem.current_position_az) data['el'].append(event.slow_data.DriveSystem.current_position_el) data['nsb_rate'] = np.array(data['nsb_rate']) * u.GHz if output is not None: table = Table(data) if os.path.isfile(output): os.remove(output) table.write(output, format='fits') time_obs = Time( np.array(data['timestamp'], dtype=np.float64) * 1e-9, format='unix' ) if plot_baselines: fig2, (ax1, ax2) = plt.subplots(2, 1, figsize=(16, 8), dpi=50) baseline_std = np.std(data['baseline'], axis=0) ax1.hist(baseline_std, 100) ax1.set_xlabel('std(baseline) [LSB]') # pixels_shown = np.arange(len(baseline_std))[baseline_std > 10] pixels_shown = [834,] ax2.plot_date( time_obs.to_datetime(), data['baseline'][:, pixels_shown], '-' ) ax2.set_xlabel('time') ax2.set_ylabel('baseline [LSB]') plt.tight_layout() plt.show() plt.close(fig2) az_obs = np.array(data['az']) * u.deg el_obs = np.array(data['el']) * u.deg n_event = len(data['timestamp']) fig1, ax = plt.subplots(1, 1, figsize=(16, 12), dpi=50) date = datetime.fromtimestamp(data['timestamp'][0]*1e-9) date_str = date.strftime("%H:%M:%S") display = CameraDisplay( DigiCam.geometry, ax=ax, norm=norm, title='NSB rate [GHz], t=' + date_str ) rate_ghz = np.array(data['nsb_rate'][0].to(u.GHz).value) display.image = rate_ghz if plot_nsb_range is None: min_range_rate = np.max([np.min(rate_ghz), 50e-3]) plot_nsb_range = (min_range_rate, np.max(rate_ghz)) display.set_limits_minmax(*plot_nsb_range) display.add_colorbar(ax=ax) bad_pixels = np.arange( len(data['good_pixels_mask'][0]) )[~data['good_pixels_mask'][0]] display.highlight_pixels(bad_pixels, color='r', linewidth=2) display.axes.set_xlim([-500., 500.]) display.axes.set_ylim([-500., 500.]) plt.tight_layout() if stars is True: stars_az, stars_alt, stars_pmag = get_stars_in_fov( az_obs[0], el_obs[0], time_obs ) stars_x, stars_y = transform_azel_to_xy( stars_az, stars_alt, az_obs, el_obs ) point_stars = [] for index_star in range(len(stars_pmag)): point_star, = ax.plot( stars_x[index_star, 0], stars_y[index_star, 0], 'ok', ms=20-2*stars_pmag[index_star], mew=3, mfc='None' ) point_stars.append(point_star) def update(i, display): print('frame', i, '/', len(data['timestamp'])) display.image = data['nsb_rate'][i].to(u.GHz).value date = datetime.fromtimestamp(data['timestamp'][i] * 1e-9) date_str = date.strftime("%H:%M:%S") display.axes.set_title('NSB rate [GHz], t=' + date_str) bad_pixels = np.arange( len(data['good_pixels_mask'][i]) )[~data['good_pixels_mask'][i]] display.highlight_pixels( bad_pixels, color='r', linewidth=2 ) if stars is True: for index_star in range(len(stars_pmag)): point_stars[index_star].set_xdata( stars_x[index_star, i] ) point_stars[index_star].set_ydata( stars_y[index_star, i] ) anim = FuncAnimation( fig1, update, frames=len(data['timestamp']), interval=20, fargs=(display, ) ) Writer = animation.writers['ffmpeg'] writer = Writer(fps=50, metadata=dict(artist='Y. Renier'), bitrate=4000, codec='h263p') output_path = os.path.dirname(plot) if plot == "show" or \ (output_path != "" and not os.path.isdir(output_path)): if not plot == "show": print('WARNING: Path ' + output_path + ' for output trigger ' + 'uniformity does not exist, displaying the plot instead.\n') display.enable_pixel_picker() plt.show() else: anim.save(plot, writer=writer) print(plot, 'created') plt.close(fig1)
def test_bad_pixels_plot(): with tempfile.TemporaryDirectory() as tmp_dir_name: plot = os.path.join(tmp_dir_name, 'test.png') get_bad_pixels(parameters_filename, plot=plot, output=None) assert os.path.isfile(plot)