def main():
args = parser.parse_args()
log.setLevel(logging.INFO)
handler = logging.StreamHandler()
logging.getLogger().addHandler(handler)
log.info(f'Input file: {args.input_file}')
log.info(f'Number of events in each subrun: {args.max_events}')
path_list = sorted(glob.glob(args.input_file))
log.info(f'list of files: {path_list}')
config_dic = {}
# read the configuration file
if args.config is not None:
config_dic = read_configuration_file(args.config)
config = Config(config_dic)
source_config = Config({
"LSTEventSource": {
"max_events": args.max_events,
"pointing_information": False,
"default_trigger_type": 'tib',
"use_flatfield_heuristic": args.use_flatfield_heuristic,
"EventTimeCalculator": {
"run_summary_path": args.run_summary_path,
},
"LSTR0Corrections": {
"drs4_pedestal_path": args.pedestal_file,
}
}
})
config.merge(source_config)
with EventSource(path_list[0]) as s:
subarray = s.subarray
timeCorr = TimeCorrectionCalculate(calib_file_path=args.output_file,
config=config,
subarray=subarray)
for i, path in enumerate(path_list):
log.info(f'File {i + 1} out of {len(path_list)}')
log.info(f'Processing: {path}')
reader = EventSource(input_url=path, config=config)
for event in tqdm(reader, disable=args.no_progress):
timeCorr.calibrate_peak_time(event)
# write output
timeCorr.finalize()
def test_TimeWaveformFitter_print(mc_gamma_testfile):
config = deepcopy(standard_config)
source = EventSource(input_url=mc_gamma_testfile,
config=Config(config["source_config"]))
subarray = source.subarray
fitter = reco.TimeWaveformFitter(subarray=subarray)
print(fitter)
def setup(self):
self.log.info('Configure EventSource...')
self.event_source = self.add_component(
EventSource.from_config(config=self.config, parent=self))
self.calibrator = self.add_component(CameraCalibrator(parent=self))
self.writer = self.add_component(
HDF5TableWriter(filename=self.outfile,
group_name='image_infos',
overwrite=True))
# Define Pre-selection for images
preselcuts = self.config['Preselect']
self.image_cutflow = CutFlow('Image preselection')
self.image_cutflow.set_cuts(
dict(no_sel=None,
n_pixel=lambda s: np.count_nonzero(s) < preselcuts['n_pixel'][
'min'],
image_amplitude=lambda q: q < preselcuts['image_amplitude'][
'min']))
# Define Pre-selection for events
self.event_cutflow = CutFlow('Event preselection')
self.event_cutflow.set_cuts(dict(no_sel=None))
def setup(self):
self.eventsource = self.add_component(
EventSource.from_url(
get_dataset_path("gamma_test_large.simtel.gz"), parent=self))
self.calibrator = self.add_component(CameraCalibrator(parent=self))
self.plotter = self.add_component(ImagePlotter(parent=self))
def setup(self):
kwargs = dict(parent=self)
self.eventsource = EventSource.from_config(**kwargs)
self.flatfield = FlatFieldCalculator.from_name(
self.flatfield_product,
**kwargs
)
self.pedestal = PedestalCalculator.from_name(
self.pedestal_product,
**kwargs
)
if self.r0calibrator_product:
self.r0calibrator = CameraR0Calibrator.from_name(
self.r0calibrator_product,
**kwargs
)
msg = "tel_id not the same for all calibration components"
assert self.r0calibrator.tel_id == self.pedestal.tel_id == self.flatfield.tel_id, msg
self.tel_id = self.flatfield.tel_id
group_name = 'tel_' + str(self.tel_id)
self.writer = HDF5TableWriter(
filename=self.output_file, group_name=group_name, overwrite=True
)
def r1_hdf5_file(r1_path):
source = EventSource(
get_dataset_path(
"gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz"),
max_events=5,
allowed_tels=[1, 2, 3, 4],
)
path = r1_path / "test_r1.h5"
writer = DataWriter(
event_source=source,
output_path=path,
write_parameters=False,
write_images=False,
write_stereo_shower=False,
write_mono_shower=False,
write_raw_waveforms=False,
write_waveforms=True,
)
for e in source:
writer(e)
writer.finish()
return path
def test_reconstructors(reconstructors):
"""
a test of the complete fit procedure on one event including:
• tailcut cleaning
• hillas parametrisation
• HillasPlane creation
• direction fit
• position fit
in the end, proper units in the output are asserted"""
filename = get_dataset_path(
"gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz"
)
source = EventSource(filename, max_events=10, focal_length_choice="nominal")
subarray = source.subarray
calib = CameraCalibrator(source.subarray)
image_processor = ImageProcessor(source.subarray)
for event in source:
calib(event)
image_processor(event)
for ReconstructorType in reconstructors:
reconstructor = ReconstructorType(subarray)
reconstructor(event)
name = ReconstructorType.__name__
assert event.dl2.stereo.geometry[name].alt.unit.is_equivalent(u.deg)
assert event.dl2.stereo.geometry[name].az.unit.is_equivalent(u.deg)
assert event.dl2.stereo.geometry[name].core_x.unit.is_equivalent(u.m)
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
event_source = EventSource(parent=self)
self.subarray = event_source.subarray
self.eventseeker = EventSeeker(event_source, parent=self)
self.calibrate = CameraCalibrator(parent=self, subarray=self.subarray)
def setup(self):
self.event_source = EventSource(parent=self)
self.event_source.allowed_tels = {self.tel}
self.calibrator = CameraCalibrator(parent=self,
subarray=self.event_source.subarray)
self.log.info(f"SELECTING EVENTS FROM TELESCOPE {self.tel}")
def setup(self):
self.log.debug(f"Open file")
self.eventsource = EventSource.from_config(parent=self)
tel_id = self.eventsource.lst_service.telescope_id
if self.eventsource.r0_r1_calibrator.drs4_pedestal_path.tel[
tel_id] is None:
raise IOError("Missing (mandatory) drs4 pedestal file in trailets")
# if data remember how many event in the files
if "LSTEventSource" in str(type(self.eventsource)):
self.tot_events = len(self.eventsource.multi_file)
self.log.debug(f"Input file has file {self.tot_events} events")
else:
self.tot_events = self.eventsource.max_events
self.simulation = True
self.processor = CalibrationCalculator.from_name(
self.calibration_product,
parent=self,
subarray=self.eventsource.subarray)
group_name = 'tel_' + str(tel_id)
self.log.debug(f"Open output file {self.output_file}")
self.writer = HDF5TableWriter(filename=self.output_file,
group_name=group_name,
overwrite=True)
def setup(self):
self.log.info('Configure EventSource...')
self.event_source = EventSource.from_config(
config=self.config,
parent=self
)
self.event_source.allowed_tels = self.config['Analysis']['allowed_tels']
self.calibrator = CameraCalibrator(
config=self.config, parent=self, eventsource=self.event_source
)
self.writer = HDF5TableWriter(
filename=self.outfile, group_name='image_infos', overwrite=True
)
# Define Pre-selection for images
preselcuts = self.config['Preselect']
self.image_cutflow = CutFlow('Image preselection')
self.image_cutflow.set_cuts(dict(
no_sel=None,
n_pixel=lambda s: np.count_nonzero(s) < preselcuts['n_pixel']['min'],
image_amplitude=lambda q: q < preselcuts['image_amplitude']['min']
))
# Define Pre-selection for events
self.event_cutflow = CutFlow('Event preselection')
self.event_cutflow.set_cuts(dict(
no_sel=None
))
def test_metadata(tmpdir: Path):
output_path = Path(tmpdir / "metadata.dl1.h5")
dataset = "lst_prod3_calibration_and_mcphotons.simtel.zst"
config = Config(
{
"DataWriter": {
"Contact": {
"name": "Maximilian Nöthe",
"email": "*****@*****.**",
"organization": "TU Dortmund",
}
}
}
)
with EventSource(get_dataset_path(dataset)) as source:
with DataWriter(
event_source=source,
output_path=output_path,
write_parameters=True,
write_images=True,
config=config,
):
pass
assert output_path.exists()
with tables.open_file(output_path) as h5file:
meta = h5file.root._v_attrs
assert meta["CTA CONTACT NAME"] == "Maximilian Nöthe"
assert meta["CTA CONTACT EMAIL"] == "*****@*****.**"
assert meta["CTA CONTACT ORGANIZATION"] == "TU Dortmund"
def setup(self):
kwargs = dict(parent=self)
self.eventsource = EventSource.from_config(**kwargs)
# remember how many event in the files
self.tot_events = len(self.eventsource.multi_file)
self.log.debug(f"Input file has file {self.tot_events} events")
self.flatfield = FlatFieldCalculator.from_name(self.flatfield_product,
**kwargs)
self.pedestal = PedestalCalculator.from_name(self.pedestal_product,
**kwargs)
if self.r0calibrator_product:
self.r0calibrator = CameraR0Calibrator.from_name(
self.r0calibrator_product, **kwargs)
msg = "tel_id not the same for all calibration components"
assert self.r0calibrator.tel_id == self.pedestal.tel_id == self.flatfield.tel_id, msg
self.tel_id = self.flatfield.tel_id
group_name = 'tel_' + str(self.tel_id)
self.log.debug(f"Open output file {self.output_file}")
self.writer = HDF5TableWriter(filename=self.output_file,
group_name=group_name,
overwrite=True)
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
self.reader = EventSource.from_config(parent=self)
self.seeker = EventSeeker(self.reader, parent=self)
self.extractor = ImageExtractor.from_name(self.extractor_product,
parent=self)
self.dl0 = CameraDL0Reducer(parent=self)
self.dl1 = CameraDL1Calibrator(extractor=self.extractor, parent=self)
self.viewer = BokehEventViewer(parent=self)
# Setup widgets
self.viewer.create()
self.viewer.enable_automatic_index_increment()
self.create_previous_event_widget()
self.create_next_event_widget()
self.create_event_index_widget()
self.create_goto_event_index_widget()
self.create_event_id_widget()
self.create_goto_event_id_widget()
self.create_telid_widget()
self.create_channel_widget()
self.create_dl1_widgets()
self.update_dl1_widget_values()
# Setup layout
self.layout = layout([[self.viewer.layout],
[
self.w_previous_event, self.w_next_event,
self.w_goto_event_index, self.w_goto_event_id
], [self.w_event_index, self.w_event_id],
[self.w_telid, self.w_channel],
[self.wb_extractor]])
def setup(self):
if self.output is None:
raise ToolConfigurationError(
"You need to provide an --output file")
if self.output.exists() and not self.overwrite:
raise ToolConfigurationError(
"Outputfile {self.output} already exists, use `--overwrite` to overwrite"
)
self.source = EventSource(parent=self)
subarray = self.source.subarray
self.calib = CameraCalibrator(subarray=subarray, parent=self)
self.ring_fitter = MuonRingFitter(parent=self)
self.intensity_fitter = MuonIntensityFitter(subarray=subarray,
parent=self)
self.cleaning = TailcutsImageCleaner(parent=self, subarray=subarray)
self.writer = HDF5TableWriter(self.output,
"",
add_prefix=True,
parent=self,
mode="w")
self.pixels_in_tel_frame = {}
self.field_of_view = {}
self.pixel_widths = {}
for p in [
"min_pixels", "pedestal", "ratio_width",
"completeness_threshold"
]:
getattr(self, p).attach_subarray(self.source.subarray)
def r1_hdf5_file(prod5_proton_simtel_path, r1_path):
source = EventSource(
prod5_proton_simtel_path,
max_events=5,
)
path = r1_path / "test_r1.h5"
writer = DataWriter(
event_source=source,
output_path=path,
write_parameters=False,
write_images=False,
write_stereo_shower=False,
write_mono_shower=False,
write_raw_waveforms=False,
write_waveforms=True,
)
for e in source:
writer(e)
writer.finish()
return path
def setup(self):
self.log.debug(f"Open file")
self.eventsource = EventSource.from_config(parent=self)
# if data remember how many event in the files
if "LSTEventSource" in str(type(self.eventsource)):
self.tot_events = len(self.eventsource.multi_file)
self.log.debug(f"Input file has file {self.tot_events} events")
else:
self.tot_events = self.eventsource.max_events
self.simulation = True
self.processor = CalibrationCalculator.from_name(
self.calibration_product,
parent=self,
subarray=self.eventsource.subarray)
if self.r0calibrator_product:
self.r0calibrator = CameraR0Calibrator.from_name(
self.r0calibrator_product, parent=self)
group_name = 'tel_' + str(self.processor.tel_id)
self.log.debug(f"Open output file {self.output_file}")
self.writer = HDF5TableWriter(filename=self.output_file,
group_name=group_name,
overwrite=True)
def setup(self):
self.eventsource = self.add_component(
EventSource.from_config(parent=self))
self.calibrator = self.add_component(
CameraCalibrator(parent=self, subarray=self.eventsource.subarray))
self.plotter = self.add_component(
ImagePlotter(subarray=self.eventsource.subarray, parent=self))
def setup(self):
print('TOLLES INFILE', self.infile)
self.event_source = EventSource.from_url(self.infile, parent=self)
self.event_source.allowed_tels = {self.tel, }
self.calibrator = CameraCalibrator(parent=self)
self.log.info(f'SELECTING EVENTS FROM TELESCOPE {self.tel}')
def setup(self):
self.source: EventSource = self.add_component(
EventSource.from_config(parent=self)
)
if self.source.input_url == '':
raise ToolConfigurationError("please specify --input <events file>")
self.calib = self.add_component(CameraCalibrator(parent=self))
self.writer = self.add_component(HDF5TableWriter(self.outfile, "muons"))
def setup(self):
print("TOLLES INFILE", self.infile)
self.event_source = EventSource.from_url(self.infile, parent=self)
self.event_source.allowed_tels = {self.tel}
self.calibrator = CameraCalibrator(parent=self,
subarray=self.event_source.subarray)
self.log.info(f"SELECTING EVENTS FROM TELESCOPE {self.tel}")
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
self.eventsource = EventSource(parent=self)
self.calibrator = CameraCalibrator(parent=self,
subarray=self.eventsource.subarray)
self.calculator = ChargeResolutionCalculator()
def test_allowed_tels_from_config():
dataset = get_dataset_path(prod5_path)
config = Config(
{"EventSource": {
"input_url": dataset,
"allowed_tels": {1, 3}
}})
reader = EventSource(config=config, parent=None)
assert reader.allowed_tels == {1, 3}
def test_from_config_default():
old_default = EventSource.input_url.default_value
dataset = get_dataset_path(prod5_path)
EventSource.input_url.default_value = dataset
config = Config()
reader = EventSource(config=config, parent=None)
assert isinstance(reader, SimTelEventSource)
assert reader.input_url == dataset
EventSource.input_url.default_value = old_default
def test_allowed_tels_from_config():
dataset = get_dataset_path("gamma_test_large.simtel.gz")
config = Config(
{"EventSource": {
"input_url": dataset,
"allowed_tels": {1, 3}
}})
reader = EventSource(config=config, parent=None)
assert len(reader.allowed_tels) == 2
def setup(self):
self.eventsource = EventSource.from_url(
get_dataset_path("gamma_test_large.simtel.gz"),
parent=self,
)
self.calibrator = CameraCalibrator(parent=self)
self.plotter = ImagePlotter(parent=self)
def test_read_subarray_description(mc_gamma_testfile, simulated_dl1_file):
from lstchain.io.io import read_subarray_description
from ctapipe.io import EventSource
source = EventSource(mc_gamma_testfile)
dl1_subarray = read_subarray_description(simulated_dl1_file)
dl1_subarray.peek()
dl1_subarray.info()
assert len(dl1_subarray.tels) == len(source.subarray.tels)
assert (dl1_subarray.to_table() == source.subarray.to_table()).all()
def setup(self):
print('TOLLES INFILE', self.infile)
self.event_source = EventSource.from_url(self.infile, parent=self)
self.event_source.allowed_tels = {
self.tel,
}
self.calibrator = CameraCalibrator(parent=self)
self.log.info(f'SELECTING EVENTS FROM TELESCOPE {self.tel}')
def test_max_events_from_config():
dataset = get_dataset_path(prod5_path)
max_events = 10
config = Config(
{"EventSource": {
"input_url": dataset,
"max_events": max_events
}})
reader = EventSource(config=config)
assert reader.max_events == max_events
def test_event_source_input_url_config_override():
dataset1 = get_dataset_path(
"gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz")
dataset2 = get_dataset_path(prod5_path)
config = Config({"EventSource": {"input_url": dataset1}})
reader = EventSource(input_url=dataset2, config=config)
assert isinstance(reader, SimTelEventSource)
assert reader.input_url == dataset2
def test_display_dl1_event(mc_gamma_testfile):
from ctapipe.io import EventSource, EventSeeker
from ctapipe.calib import CameraCalibrator
source = EventSource(mc_gamma_testfile, back_seekable=True)
seeker = EventSeeker(source)
event = seeker.get_event_index(11) # event 11 has telescopes 1 and 4 with data
CameraCalibrator(subarray=source.subarray)(event)
display_dl1_event(event, source.subarray.tel[1].camera.geometry, tel_id=1)
display_dl1_event(event, source.subarray.tel[4].camera.geometry, tel_id=4)
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
event_source = self.add_component(EventSource.from_config(parent=self))
self.eventseeker = self.add_component(
EventSeeker(event_source, parent=self))
self.extractor = self.add_component(
ImageExtractor.from_name(self.extractor_product, parent=self))
self.calibrate = self.add_component(
CameraCalibrator(parent=self, image_extractor=self.extractor))
def setup(self):
self.log.info("Configure EventSource...")
self.event_source = EventSource.from_url(self.infile, parent=self)
self.calibrator = CameraCalibrator(subarray=self.event_source.subarray,
parent=self)
self.writer = HDF5TableWriter(filename=self.outfile,
group_name="image_infos",
overwrite=True,
parent=self)
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
event_source = EventSource.from_config(parent=self)
self.eventseeker = EventSeeker(event_source, parent=self)
self.extractor = ImageExtractor.from_name(
self.extractor_product,
parent=self,
)
self.dl0 = CameraDL0Reducer(parent=self)
self.dl1 = CameraDL1Calibrator(extractor=self.extractor, parent=self)
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
self.reader = EventSource.from_config(parent=self)
self.seeker = EventSeeker(self.reader, parent=self)
self.extractor = ImageExtractor.from_name(
self.extractor_product,
parent=self
)
self.r1 = CameraR1Calibrator.from_eventsource(
eventsource=self.reader,
parent=self
)
self.dl0 = CameraDL0Reducer(parent=self)
self.dl1 = CameraDL1Calibrator(
extractor=self.extractor,
parent=self
)
self.viewer = BokehEventViewer(parent=self)
# Setup widgets
self.viewer.create()
self.viewer.enable_automatic_index_increment()
self.create_previous_event_widget()
self.create_next_event_widget()
self.create_event_index_widget()
self.create_goto_event_index_widget()
self.create_event_id_widget()
self.create_goto_event_id_widget()
self.create_telid_widget()
self.create_channel_widget()
self.create_dl1_widgets()
self.update_dl1_widget_values()
# Setup layout
self.layout = layout([
[self.viewer.layout],
[
self.w_previous_event,
self.w_next_event,
self.w_goto_event_index,
self.w_goto_event_id
],
[self.w_event_index, self.w_event_id],
[self.w_telid, self.w_channel],
[self.wb_extractor]
])
