def test_pointing_info():
from ctapipe_io_lst import LSTEventSource
# test source works when not requesting pointing info
with LSTEventSource(
test_r0_dir / 'LST-1.1.Run02008.0000_first50.fits.fz',
apply_drs4_corrections=False,
pointing_information=False,
max_events=1
) as source:
for e in source:
assert np.isnan(e.pointing.tel[1].azimuth)
# test we get an error when requesting pointing info but nor drive report given
with pytest.raises(ValueError):
with LSTEventSource(
test_r0_dir / 'LST-1.1.Run02008.0000_first50.fits.fz',
apply_drs4_corrections=False,
max_events=1
) as source:
next(iter(source))
config = {
'LSTEventSource': {
'apply_drs4_corrections': False,
'max_events': 1,
'PointingSource': {
'drive_report_path': str(test_drive_report)
},
},
}
with LSTEventSource(
test_r0_dir / 'LST-1.1.Run02008.0000_first50.fits.fz',
config=Config(config),
) as source:
for e in source:
# Tue Feb 18 21:03:09 2020 1582059789 Az 197.318 197.287 197.349 0 El 7.03487 7.03357 7.03618 0.0079844 RA 83.6296 Dec 22.0144
assert u.isclose(e.pointing.array_ra, 83.6296 * u.deg)
assert u.isclose(e.pointing.array_dec, 22.0144 * u.deg)
expected_alt = (90 - 7.03487) * u.deg
assert u.isclose(e.pointing.tel[1].altitude.to(u.deg), expected_alt, rtol=1e-2)
expected_az = 197.318 * u.deg
assert u.isclose(e.pointing.tel[1].azimuth.to(u.deg), expected_az, rtol=1e-2)
def setup(self):
self.output_path = self.output_path.expanduser().resolve()
if self.output_path.exists():
if self.overwrite:
self.log.warning("Overwriting %s", self.output_path)
self.output_path.unlink()
else:
raise ToolConfigurationError(
f"Output file {self.output_path} exists"
", use the `overwrite` option or choose another `output_path` "
)
self.log.debug("output path: %s", self.output_path)
Provenance().add_output_file(str(self.output_path), role="DL1/Event")
self.source = LSTEventSource(
parent=self,
pointing_information=False,
trigger_information=False,
)
# set some config options, these are necessary for this tool,
# so we set them here and not via the config system
self.source.r0_r1_calibrator.r1_sample_start = 0
self.source.r0_r1_calibrator.r1_sample_end = N_SAMPLES
self.source.r0_r1_calibrator.offset = 0
self.source.r0_r1_calibrator.apply_spike_correction = False
self.source.r0_r1_calibrator.apply_timelapse_correction = True
self.source.r0_r1_calibrator.apply_drs4_pedestal_correction = False
n_stats = N_GAINS * N_PIXELS * N_CAPACITORS_PIXEL
self.baseline_stats = OnlineStats(n_stats)
self.spike0_stats = OnlineStats(n_stats)
self.spike1_stats = OnlineStats(n_stats)
self.spike2_stats = OnlineStats(n_stats)
def test_get_first_capacitor():
from ctapipe_io_lst import LSTEventSource
from ctapipe_io_lst.calibration import (
get_first_capacitors_for_pixels,
N_GAINS, N_PIXELS_MODULE, N_MODULES,
)
tel_id = 1
source = LSTEventSource(
test_r0_calib_path,
apply_drs4_corrections=False,
pointing_information=False,
)
event = next(iter(source))
first_capacitor_id = event.lst.tel[tel_id].evt.first_capacitor_id
with tables.open_file(resource_dir / 'first_caps.hdf5', 'r') as f:
expected = f.root.first_capacitor_for_modules[:]
first_caps = get_first_capacitors_for_pixels(first_capacitor_id)
# we used a different shape (N_MODULES, N_GAINS, N_PIXELS_MODULE) before
# so have to reshape to be able to compare
first_caps = first_caps.reshape((N_GAINS, N_MODULES, N_PIXELS_MODULE))
first_caps = np.swapaxes(first_caps, 0, 1)
assert np.all(first_caps == expected)
def test_no_gain_selection_no_drs4time_calib():
from ctapipe_io_lst import LSTEventSource
from ctapipe_io_lst.constants import N_PIXELS, N_GAINS, N_SAMPLES
config = Config({
'LSTEventSource': {
'pointing_information': False,
'LSTR0Corrections': {
'drs4_pedestal_path': test_drs4_pedestal_path,
'calibration_path': test_calib_path,
'select_gain': False,
},
},
})
source = LSTEventSource(
input_url=test_r0_calib_path,
config=config,
)
assert source.r0_r1_calibrator.mon_data is not None
with source:
for event in source:
assert event.r1.tel[1].waveform is not None
assert event.r1.tel[1].waveform.ndim == 3
assert event.r1.tel[1].waveform.shape == (N_GAINS, N_PIXELS, N_SAMPLES - 4)
def test_init():
from ctapipe_io_lst import LSTEventSource
from ctapipe_io_lst.calibration import LSTR0Corrections
subarray = LSTEventSource.create_subarray(geometry_version=4)
r0corr = LSTR0Corrections(subarray)
assert r0corr.last_readout_time.keys() == {1, }
def main():
log.setLevel(logging.INFO)
handler = logging.StreamHandler()
logging.getLogger().addHandler(handler)
log.info(f'Input file: {args.input_file}')
log.info(f'Number of events in each subrun: {args.max_events}')
path_list = sorted(glob.glob(args.input_file))
log.info(f'list of files: {path_list}')
config_dic = {}
if args.config_file is not None:
try:
config_dic = read_configuration_file(args.config_file)
except ("Custom configuration could not be loaded !!!"):
pass
# read the configuration file
config = Config(config_dic)
# declare the pedestal calibrator
lst_r0 = LSTR0Corrections(pedestal_path=args.pedestal_file, config=config)
reader = LSTEventSource(input_url=path_list[0], max_events=args.max_events)
# declare the time corrector
timeCorr = TimeCorrectionCalculate(calib_file_path=args.output_file,
config=config,
subarray=reader.subarray)
tel_id = timeCorr.tel_id
for i, path in enumerate(path_list):
log.info(f'File {i+1} out of {len(path_list)}')
log.info(f'Processing: {path}')
reader = LSTEventSource(input_url=path, max_events=args.max_events)
for event in reader:
if event.index.event_id % 5000 == 0:
log.info(f'event id = {event.index.event_id}')
lst_r0.calibrate(event)
# Cut in signal to avoid cosmic events
if event.r1.tel[tel_id].trigger_type == 4 or (np.median(
np.sum(event.r1.tel[tel_id].waveform[0], axis=1)) > 300):
timeCorr.calibrate_peak_time(event)
# write output
timeCorr.finalize()
def test_len():
from ctapipe_io_lst import LSTEventSource
with LSTEventSource(
test_r0_dir / 'LST-1.1.Run02008.0000_first50.fits.fz',
apply_drs4_corrections=False,
pointing_information=False,
) as source:
assert len(source) == 200
with LSTEventSource(
test_r0_dir / 'LST-1.1.Run02008.0000_first50.fits.fz',
pointing_information=False,
apply_drs4_corrections=False,
max_events=10,
) as source:
assert len(source) == 10
def test_no_reference_values_no_ucts(caplog):
'''
Test extracting the reference counters from the first event without
ucts.
'''
caplog.set_level(logging.CRITICAL)
from ctapipe_io_lst.event_time import EventTimeCalculator, CENTRAL_MODULE
from ctapipe_io_lst import LSTEventSource
subarray = LSTEventSource.create_subarray(geometry_version=4, tel_id=1)
# test ucts reference extaction works
time_calculator = EventTimeCalculator(
subarray=subarray,
run_id=1,
expected_modules_id=np.arange(N_MODULES),
extract_reference=True,
)
# fill an artifical event with just enough information so we can test this
first_event_time = Time.now()
delay = 5 * u.s
run_start = first_event_time - delay
tel_id = 1
event = LSTArrayEventContainer()
event.index.event_id = 1
lst = event.lst.tel[tel_id]
lst.svc.module_ids = np.arange(N_MODULES)
lst.evt.extdevices_presence = 0b1111_1101
lst.evt.pps_counter = np.full(N_MODULES, 100)
lst.evt.tenMHz_counter = np.zeros(N_MODULES)
lst.evt.tenMHz_counter[CENTRAL_MODULE] = 2
lst.svc.date = run_start.unix
lst.evt.module_status = np.ones(N_MODULES)
# actually test it
time = time_calculator(tel_id, event)
assert np.isclose((first_event_time - time - delay).to_value(u.us),
0,
atol=0.5)
# test that we got the critical log message with the reference values
found = False
for record in caplog.records:
if 'timestamp: ' in record.message and 'counter: ' in record.message:
found = True
assert found
# test error if not first subrun
time_calculator = EventTimeCalculator(
subarray=subarray,
run_id=1,
expected_modules_id=np.arange(N_MODULES),
extract_reference=True)
event.index.event_id = 100001
with pytest.raises(ValueError):
time_calculator(tel_id, event)
def test_gain_selected():
from ctapipe_io_lst import LSTEventSource
config = Config(dict(
LSTEventSource=dict(
default_trigger_type='tib', # ucts unreliable in this run
apply_drs4_corrections=True,
pointing_information=False,
LSTR0Corrections=dict(
apply_drs4_pedestal_correction=False,
apply_spike_correction=False,
apply_timelapse_correction=False,
offset=400,
)
)
))
source = LSTEventSource(
test_r0_dir / 'LST-1.1.Run02008.0000_first50_gainselected.fits.fz',
config=config,
)
original_source = LSTEventSource(
test_r0_dir / 'LST-1.1.Run02008.0000_first50.fits.fz',
config=config,
)
gain_selector = ThresholdGainSelector(threshold=3500)
for event, original_event in zip(source, original_source):
if event.trigger.event_type in {EventType.FLATFIELD, EventType.SKY_PEDESTAL}:
assert event.r0.tel[1].waveform is not None
assert event.r0.tel[1].waveform.shape == (N_GAINS, N_PIXELS, N_SAMPLES)
assert event.r1.tel[1].waveform is not None
assert event.r1.tel[1].waveform.shape == (N_GAINS, N_PIXELS, N_SAMPLES - 4)
else:
if event.r0.tel[1].waveform is not None:
assert event.r0.tel[1].waveform.shape == (N_GAINS, N_PIXELS, N_SAMPLES)
assert event.r1.tel[1].waveform.shape == (N_PIXELS, N_SAMPLES - 4)
# compare to original file
selected_gain = gain_selector(original_event.r1.tel[1].waveform)
pixel_idx = np.arange(N_PIXELS)
waveform = original_event.r1.tel[1].waveform[selected_gain, pixel_idx]
assert np.allclose(event.r1.tel[1].waveform, waveform)
assert event.count == 199
def remove_star_and_run(self, list_of_file, max_events,
noise_pixels_id_list):
signal_place_after_clean = np.zeros(1855)
sum_ped_ev = 0
alive_ped_ev = 0
for input_file in list_of_file:
print(input_file)
r0_r1_calibrator = LSTR0Corrections(pedestal_path=None,
r1_sample_start=3,
r1_sample_end=39)
reader = LSTEventSource(input_url=input_file,
max_events=max_events)
for i, ev in enumerate(reader):
r0_r1_calibrator.calibrate(ev)
if i % 10000 == 0:
print(ev.r0.event_id)
if ev.lst.tel[1].evt.tib_masked_trigger == 32:
sum_ped_ev += 1
self.r1_dl1_calibrator(ev)
img = ev.dl1.tel[1].image
img[noise_pixels_id_list] = 0
geom = ev.inst.subarray.tel[1].camera
clean = tailcuts_clean(geom, img,
**self.cleaning_parameters)
cleaned = img.copy()
cleaned[~clean] = 0.0
signal_place_after_clean[np.where(clean == True)] += 1
if np.sum(cleaned > 0) > 0:
alive_ped_ev += 1
fig, ax = plt.subplots(figsize=(10, 8))
geom = ev.inst.subarray.tel[1].camera
disp0 = CameraDisplay(geom, ax=ax)
disp0.image = signal_place_after_clean / sum_ped_ev
disp0.highlight_pixels(noise_pixels_id_list, linewidth=3)
disp0.add_colorbar(ax=ax,
label="N times signal remain after cleaning [%]")
disp0.cmap = 'gnuplot2'
ax.set_title("{} \n {}/{}".format(
input_file.split("/")[-1][8:21], alive_ped_ev, sum_ped_ev),
fontsize=25)
print("{}/{}".format(alive_ped_ev, sum_ped_ev))
ax.set_xlabel(" ")
ax.set_ylabel(" ")
plt.tight_layout()
plt.show()
def check_interleave_pedestal_cleaning(self, list_of_file, max_events,
sigma, dl1_file):
high_gain = 0
ped_mean_pe, ped_rms_pe = get_bias_and_rms(dl1_file)
bad_pixel_ids = np.where(ped_rms_pe[1, high_gain, :] == 0)[0]
print(bad_pixel_ids)
th = get_threshold(ped_mean_pe[1, high_gain, :],
ped_rms_pe[1, high_gain, :], sigma)
make_camera_binary_image(th, sigma,
self.cleaning_parameters['picture_thresh'],
bad_pixel_ids)
signal_place_after_clean = np.zeros(1855)
sum_ped_ev = 0
alive_ped_ev = 0
for input_file in list_of_file:
print(input_file)
r0_r1_calibrator = LSTR0Corrections(pedestal_path=None,
r1_sample_start=3,
r1_sample_end=39)
reader = LSTEventSource(input_url=input_file,
max_events=max_events)
for i, ev in enumerate(reader):
r0_r1_calibrator.calibrate(ev)
if i % 10000 == 0:
print(ev.r0.event_id)
if ev.lst.tel[1].evt.tib_masked_trigger == 32:
sum_ped_ev += 1
self.r1_dl1_calibrator(ev)
img = ev.dl1.tel[1].image
img[bad_pixel_ids] = 0
geom = ev.inst.subarray.tel[1].camera
clean = tailcuts_pedestal_clean(geom, img, th,
**self.cleaning_parameters)
cleaned = img.copy()
cleaned[~clean] = 0.0
signal_place_after_clean[np.where(clean == True)] += 1
if np.sum(cleaned > 0) > 0:
alive_ped_ev += 1
noise_remain = signal_place_after_clean / sum_ped_ev
self.plot_camera_display(noise_remain, input_file, bad_pixel_ids,
alive_ped_ev, sum_ped_ev)
def test_already_gain_selected():
from ctapipe_io_lst import LSTEventSource
config = Config({
'LSTEventSource': {
'pointing_information': False,
'LSTR0Corrections': {
'drs4_pedestal_path': test_drs4_pedestal_path,
'drs4_time_calibration_path': test_time_calib_path,
'calibration_path': test_calib_path,
},
},
})
source = LSTEventSource(test_r0_gainselected_path, config=config)
reference_source = LSTEventSource(test_r0_path, config=config)
with source, reference_source:
for event, reference_event in zip(source, reference_source):
assert np.all(event.r1.tel[1].waveform == reference_event.r1.tel[1].waveform)
assert event.count == 199
def test_subarray():
from ctapipe_io_lst import LSTEventSource
source = LSTEventSource(test_r0_path)
subarray = source.subarray
subarray.info()
subarray.to_table()
assert source.lst_service.telescope_id == 1
assert source.lst_service.num_modules == 265
with tempfile.NamedTemporaryFile(suffix='.h5') as f:
subarray.to_hdf(f.name)
def test_missing_modules():
from ctapipe_io_lst import LSTEventSource
source = LSTEventSource(
test_missing_module_path,
apply_drs4_corrections=False,
pointing_information=False,
)
assert source.lst_service.telescope_id == 1
assert source.lst_service.num_modules == 264
fill = np.iinfo(np.uint16).max
for event in source:
# one module missing, so 7 pixels
assert np.count_nonzero(event.mon.tel[1].pixel_status.hardware_failing_pixels) == N_PIXELS_MODULE * N_GAINS
assert np.count_nonzero(event.r0.tel[1].waveform == fill) == N_PIXELS_MODULE * N_SAMPLES * N_GAINS
# 514 is one of the missing pixels
assert np.all(event.r0.tel[1].waveform[:, 514] == fill)
def test_multifile():
from ctapipe_io_lst import LSTEventSource
event_count = 0
with LSTEventSource(
input_url=test_r0_path_all_streams,
apply_drs4_corrections=False,
pointing_information=False,
) as source:
assert len(set(source.file_list)) == 4
assert len(source) == 200
for event in source:
event_count += 1
# make sure all events are present and in the correct order
assert event.index.event_id == event_count
# make sure we get all events from all streams (50 per stream)
assert event_count == 200
def test_loop_over_events():
from ctapipe_io_lst import LSTEventSource
n_events = 10
inputfile_reader = LSTEventSource(input_url=example_file_path,
max_events=n_events)
for i, event in enumerate(inputfile_reader):
assert event.r0.tels_with_data == [0]
for telid in event.r0.tels_with_data:
assert event.r0.event_id == FIRST_EVENT_NUMBER_IN_FILE + i
n_gain = 2
n_camera_pixels = event.inst.subarray.tels[telid].camera.n_pixels
num_samples = event.lst.tel[telid].svc.num_samples
waveform_shape = (n_gain, n_camera_pixels, num_samples)
assert event.r0.tel[telid].waveform.shape == waveform_shape
# make sure max_events works
assert i == n_events - 1
def test_loop_over_events():
from ctapipe_io_lst import LSTEventSource
n_events = 10
source = LSTEventSource(input_url=example_file_path, max_events=n_events)
for i, event in enumerate(source, start=1):
assert event.r0.tels_with_data == [0]
for telid in event.r0.tels_with_data:
assert event.index.event_id == i
n_gain = 2
n_camera_pixels = \
source.subarray.tels[telid].camera.geometry.n_pixels
num_samples = event.lst.tel[telid].svc.num_samples
waveform_shape = (n_gain, n_camera_pixels, num_samples)
assert event.r0.tel[telid].waveform.shape == waveform_shape
# make sure max_events works
assert i == n_events
def test_source_with_drs4_pedestal():
from ctapipe_io_lst import LSTEventSource
config = Config({
'LSTEventSource': {
'pointing_information': False,
'LSTR0Corrections': {
'drs4_pedestal_path': test_drs4_pedestal_path,
},
},
})
source = LSTEventSource(
input_url=test_r0_path,
config=config,
)
assert source.r0_r1_calibrator.drs4_pedestal_path.tel[1] == test_drs4_pedestal_path.absolute()
with source:
for event in source:
assert event.r1.tel[1].waveform is not None
def test_loop_over_events():
from ctapipe_io_lst import LSTEventSource
n_events = 10
source = LSTEventSource(
input_url=test_r0_path,
max_events=n_events,
apply_drs4_corrections=False,
pointing_information=False,
)
for i, event in enumerate(source):
assert event.count == i
for telid in event.r0.tel.keys():
n_gains = 2
n_pixels = source.subarray.tels[telid].camera.geometry.n_pixels
n_samples = event.lst.tel[telid].svc.num_samples
waveform_shape = (n_gains, n_pixels, n_samples)
assert event.r0.tel[telid].waveform.shape == waveform_shape
assert event.mon.tel[telid].pixel_status.hardware_failing_pixels.shape == (n_gains, n_pixels)
# make sure max_events works
assert (i + 1) == n_events
def test_interpolation():
from ctapipe_io_lst.pointing import PointingSource
from ctapipe_io_lst import LSTEventSource
subarray = LSTEventSource.create_subarray(geometry_version=4)
pointing_source = PointingSource(
subarray=subarray,
drive_report_path=test_drive_report,
)
time = Time('2020-02-18T21:40:21')
# El is really zenith distance
# Tue Feb 18 21:40:20 2020 1582062020 Az 230.834 230.819 230.849 7.75551 El 10.2514 10.2485 10.2543 0.00948548 RA 86.6333 Dec 22.0144
# Tue Feb 18 21:40:23 2020 1582062022 Az 230.896 230.881 230.912 9.03034 El 10.2632 10.2603 10.2661 0.00948689 RA 86.6333 Dec 22.0144
pointing = pointing_source.get_pointing_position_altaz(tel_id=1, time=time)
expected_alt = (90 - 0.5 * (10.2514 + 10.2632)) * u.deg
assert u.isclose(pointing.altitude, expected_alt)
assert u.isclose(pointing.azimuth, 0.5 * (230.834 + 230.896) * u.deg)
ra, dec = pointing_source.get_pointing_position_icrs(tel_id=1, time=time)
assert u.isclose(ra, 86.6333 * u.deg)
assert u.isclose(dec, 22.0144 * u.deg)
def test_missing_module():
from ctapipe_io_lst import LSTEventSource
from ctapipe_io_lst.constants import N_PIXELS_MODULE, N_SAMPLES
config = Config({
'LSTEventSource': {
'pointing_information': False,
'LSTR0Corrections': {
'drs4_pedestal_path': test_drs4_pedestal_path,
'drs4_time_calibration_path': test_time_calib_path,
'calibration_path': test_calib_path,
},
},
})
source = LSTEventSource(
input_url=test_missing_module_path,
config=config,
)
assert source.r0_r1_calibrator.mon_data is not None
with source:
for event in source:
waveform = event.r1.tel[1].waveform
assert waveform is not None
failing_pixels = event.mon.tel[1].pixel_status.hardware_failing_pixels
# one module failed, in each gain channel
assert np.count_nonzero(failing_pixels) == 2 * N_PIXELS_MODULE
# there might be zeros in other pixels than just the broken ones
assert np.count_nonzero(waveform == 0) >= N_PIXELS_MODULE * (N_SAMPLES - 4)
# waveforms in failing pixels must be all 0
assert np.all(waveform[failing_pixels[HIGH_GAIN]] == 0)
def test_source_with_all():
from ctapipe_io_lst import LSTEventSource
config = Config({
'LSTEventSource': {
'pointing_information': False,
'LSTR0Corrections': {
'drs4_pedestal_path': test_drs4_pedestal_path,
'drs4_time_calibration_path': test_time_calib_path,
'calibration_path': test_calib_path,
},
},
})
source = LSTEventSource(
input_url=test_r0_path,
config=config,
)
assert source.r0_r1_calibrator.mon_data is not None
with source:
for event in source:
assert event.r1.tel[1].waveform is not None
assert np.any(event.calibration.tel[1].dl1.time_shift != 0)
def check_interleave_pedestal_cleaning(path_list, calib_time_file, calib_file, max_events=10000):
signal_place_after_clean = np.zeros(1855)
sum_ped_ev = 0
alive_ped_ev = 0
for path in path_list:
print(path)
r0_r1_calibrator = LSTR0Corrections(pedestal_path=None,
r1_sample_start=3,
r1_sample_end=39)
r1_dl1_calibrator = LSTCameraCalibrator(calibration_path=calib_file,
time_calibration_path=calib_time_file,
extractor_product="LocalPeakWindowSum",
config=charge_config,
allowed_tels=[1])
reader = LSTEventSource(input_url=path, max_events=max_events)
for i, ev in enumerate(reader):
r0_r1_calibrator.calibrate(ev)
if i%10000 == 0:
print(ev.r0.event_id)
if ev.lst.tel[1].evt.tib_masked_trigger == 32:
sum_ped_ev += 1
r1_dl1_calibrator(ev)
img = ev.dl1.tel[1].image
geom = ev.inst.subarray.tel[1].camera
clean = tailcuts_clean(
geom,
img,
picture_thresh=6,
boundary_thresh=3,
min_number_picture_neighbors=1,
keep_isolated_pixels=False
)
cleaned = img.copy()
cleaned[~clean] = 0.0
signal_place_after_clean[np.where(clean == True)] += 1
if np.sum(cleaned>0) > 0:
alive_ped_ev += 1
fig, ax = plt.subplots(figsize=(8, 8))
geom = ev.inst.subarray.tel[1].camera
disp0 = CameraDisplay(geom, ax=ax)
disp0.image = signal_place_after_clean/sum_ped_ev
disp0.add_colorbar(ax=ax, label="N times signal remain after cleaning")
disp0.cmap = 'gnuplot2'
ax.set_title("{} \n {}/{}".format(path.split("/")[-1][8:21], alive_ped_ev, sum_ped_ev))
print(path.split("/")[-1][8:21])
print("{}/{}".format(alive_ped_ev, sum_ped_ev))
ax.set_xlabel(" ")
ax.set_ylabel(" ")
plt.tight_layout()
plt.show()
return signal_place_after_clean, sum_ped_ev
parser.add_argument("--output_file",
help="Path where script create pedestal file",
type=str)
parser.add_argument("--max_events",
help="Maximum numbers of events to read",
type=int,
default=5000)
args = parser.parse_args()
if __name__ == '__main__':
print("input file: {}".format(args.input_file))
print("max events: {}".format(args.max_events))
reader = LSTEventSource(input_url=args.input_file,
max_events=args.max_events)
print("---> Number of files", reader.multi_file.num_inputs())
seeker = EventSeeker(reader)
ev = seeker[0]
n_modules = ev.lst.tel[0].svc.num_modules
telid = ev.r0.tels_with_data[0]
config = Config({"LSTR0Corrections": {"tel_id": telid}})
lst_r0 = LSTR0Corrections(config=config)
ped = DragonPedestal(telid=telid)
PedList = []
pedestal_value_array = np.zeros((n_modules, 2, 7, 4096), dtype=np.uint16)
for i in range(0, n_modules):
def test_is_compatible():
from ctapipe_io_lst import LSTEventSource
assert LSTEventSource.is_compatible(example_file_path)
def test_ucts_jumps():
'''
We creat toy data that will have two ucts jumps.
When a ucts event goes missing, the event actually contains
the ucts data for the next event. For a second jump, the events are
no out of sync by two.
We are going to just use event ids for the timestamps
'''
from ctapipe_io_lst.event_time import EventTimeCalculator
from ctapipe_io_lst import LSTEventSource
tel_id = 1
event = LSTArrayEventContainer()
lst = event.lst.tel[tel_id]
lst.evt.extdevices_presence = 0b1111_1111
lst.svc.module_ids = np.arange(N_MODULES)
lst.evt.module_status = np.ones(N_MODULES)
subarray = LSTEventSource.create_subarray(geometry_version=4, tel_id=1)
true_time_s = int(Time.now().unix)
s_to_ns = int(1e9)
time_calculator = EventTimeCalculator(
subarray=subarray,
run_id=1,
expected_modules_id=np.arange(N_MODULES),
dragon_reference_time=true_time_s * s_to_ns,
dragon_reference_counter=0,
dragon_module_id=1,
timestamp=
'ucts' # use ucts to make sure we identify jumps and fallback to tib
)
n_events = 22
true_event_id = np.arange(n_events)
# One event every 10 us.
true_time_ns = np.uint64(10 * true_event_id * int(1e3))
# no jumps
table = Table({
'event_id':
true_event_id,
'ucts_timestamp':
np.uint64(true_time_s * s_to_ns + true_time_ns),
'tib_pps_counter':
np.full(n_events, 0),
'tib_tenMHz_counter': (true_time_ns / 100).astype(np.uint64),
'pps_counter':
[np.full(N_MODULES, 0, dtype=np.uint32) for _ in range(n_events)],
'tenMHz_counter':
[np.full(N_MODULES, t // 100, dtype=np.uint32) for t in true_time_ns],
# to check if we handle jumps correctly, we put the event id here
'ucts_trigger_type':
np.arange(n_events),
'tib_masked_trigger':
np.arange(n_events),
})
for i in range(n_events):
for col in table.colnames:
setattr(lst.evt, col, table[col][i])
time_calculator(tel_id, event)
assert len(time_calculator.previous_ucts_timestamps[tel_id]) == 0
# no we introduce three jumps
for col in ('ucts_timestamp', 'ucts_trigger_type'):
table[col][5:-1] = table[col][6:]
table[col][12:-1] = table[col][13:]
table[col][15:-1] = table[col][16:]
table = table[:-3]
ucts_trigger_types = []
last_time = None
for i in range(n_events - 3):
for col in table.colnames:
setattr(lst.evt, col, table[col][i])
event.index.event_id = table['event_id'][i]
event.lst.tel[tel_id].evt.ucts_jump = False
time = time_calculator(tel_id, event)
if last_time is not None:
# timestamp is only accurate to 1 us due to floating point
assert np.isclose((time - last_time).to_value(u.us), 10, atol=0.5)
last_time = time
if i < 5:
assert len(time_calculator.previous_ucts_timestamps[tel_id]) == 0
elif i < 13:
assert len(time_calculator.previous_ucts_timestamps[tel_id]) == 1
elif i < 17:
assert len(time_calculator.previous_ucts_timestamps[tel_id]) == 2
if i in {5, 13, 17}:
assert event.lst.tel[
tel_id].evt.ucts_jump, f'jump not found in {i}'
else:
assert not event.lst.tel[
tel_id].evt.ucts_jump, f'unexpected jump in {i}'
ucts_trigger_types.append(lst.evt.ucts_trigger_type)
assert np.all(np.array(ucts_trigger_types) == table['event_id'])
def lst1_subarray():
return LSTEventSource.create_subarray(geometry_version=4)
default="")
parser.add_argument("--output_file",
help="Path to output plot in png format.",
type=str,
default="")
args = parser.parse_args()
if __name__ == '__main__':
print("input file: {}".format(args.input_file))
run_path = args.input_file
print("Run path: {}".format(run_path))
reader = LSTEventSource(input_url=run_path)
hist_r0_hg = Histogram(ranges=[0, 1000], nbins=(1000))
hist_r0_lg = Histogram(ranges=[0, 1000], nbins=(1000))
for i, ev in enumerate(reader):
if i%5 == 0: # histogram is filled every 5 event
hist_r0_hg.fill(ev.r0.tel[1].waveform[0, :, 2:38].ravel())
hist_r0_lg.fill(ev.r0.tel[1].waveform[1, :, 2:38].ravel())
mean_hg, rms_hg = get_mean_rms(hist_r0_hg)
mean_lg, rms_lg = get_mean_rms(hist_r0_lg)
print("RMS HG = {:.2f} ADC".format(rms_hg))
print("RMS LG = {:.2f} ADC".format(rms_lg))
def type_of_run(date_path, run_number, counters, n_events=500):
"""
Guessing empirically the type of run based on the percentage of
pedestals/mono trigger types from the first n_events:
DRS4 pedestal run (DRS4): 100% mono events (trigger_type == 1)
cosmic data run (DATA): <10% pedestal events (trigger_type == 32)
pedestal-calibration run (PEDCALIB): ~50% mono, ~50% pedestal events
Otherwise (ERROR) the run is not expected to be processed.
This method may not give always the correct type.
At some point this should be taken directly from TCU.
Parameters
----------
date_path : pathlib.Path
Path to the R0 files
run_number : int
Run id
counters : dict
Dict containing the reference counters and timestamps
n_events : int
Number of events used to infer the type of the run
Returns
-------
run_type: str
Type of run (DRS4, PEDCALIB, DATA, ERROR)
"""
pattern = f"LST-1.1.Run{run_number:05d}.0000*.fits.fz"
list_of_files = sorted(date_path.glob(pattern))
if len(list_of_files) == 0:
log.error(f"First subrun not found for {pattern}")
return "ERROR"
filename = list_of_files[0]
config = Config()
config.LSTEventSource.apply_drs4_corrections = False
config.LSTEventSource.pointing_information = False
config.EventTimeCalculator.dragon_reference_time = int(counters["dragon_reference_time"])
config.EventTimeCalculator.dragon_reference_counter = int(counters["dragon_reference_counter"])
config.EventTimeCalculator.dragon_module_id = int(counters["dragon_reference_module_id"])
try:
with LSTEventSource(filename, config=config, max_events=n_events) as source:
source.log.setLevel(logging.ERROR)
event_type_counts = Counter(event.trigger.event_type for event in source)
n_pedestals = event_type_counts[EventType.SKY_PEDESTAL]
n_subarray = event_type_counts[EventType.SUBARRAY]
if n_subarray / n_events > 0.999:
run_type = "DRS4"
elif n_pedestals / n_events > 0.1:
run_type = "PEDCALIB"
elif n_pedestals / n_events < 0.1:
run_type = "DATA"
else:
run_type = "ERROR"
except (AttributeError, ValueError, IOError, IndexError) as err:
log.error(f"File {filename} has error: {err!r}")
run_type = "ERROR"
return run_type
def test_is_compatible():
from ctapipe_io_lst import LSTEventSource
assert LSTEventSource.is_compatible(test_r0_path)
