def test_targetio_calibrator(): pytest.importorskip("target_calib") url_r0 = get_dataset_path("targetmodule_r0.tio") url_r1 = get_dataset_path("targetmodule_r1.tio") pedpath = get_dataset_path("targetmodule_ped.tcal") source_r0 = TargetIOEventSource(input_url=url_r0) source_r1 = TargetIOEventSource(input_url=url_r1) r1c = CameraR1CalibratorFactory.produce(eventsource=source_r0) event_r0 = source_r0._get_event_by_index(0) event_r1 = source_r1._get_event_by_index(0) r1c.calibrate(event_r0) assert_array_equal(event_r0.r0.tel[0].waveform, event_r0.r1.tel[0].waveform) r1c = CameraR1CalibratorFactory.produce( eventsource=source_r0, pedestal_path=pedpath ) r1c.calibrate(event_r0) assert_array_almost_equal(event_r0.r1.tel[0].waveform, event_r1.r1.tel[0].waveform, 1)
def test_event_source_input_url_config_override(): dataset1 = get_dataset_path("gamma_test_large.simtel.gz") dataset2 = get_dataset_path("gamma_test_large.simtel.gz") config = Config({'EventSource': {'input_url': dataset1}}) reader = event_source(input_url=dataset2, config=config) assert isinstance(reader, SimTelEventSource) assert reader.input_url == dataset2
def test_geom(): dataset = get_dataset_path("chec_r1.tio") with TargetIOEventSource(input_url=dataset) as source: event = source._get_event_by_index(0) assert event.inst.subarray.tels[0].camera.pix_x.size == 2048 dataset = get_dataset_path("targetmodule_r1.tio") with TargetIOEventSource(input_url=dataset) as source: event = source._get_event_by_index(0) assert event.inst.subarray.tels[0].camera.pix_x.size == 64
def get_array_layout(instrument_name): """ Returns the array layout for the given instrument as an `astropy.table.Table` object. """ name = instrument_name.lower() try: layoutfile = get_dataset_path(f'{name}_arraylayout.fits') except KeyError: layoutfile = get_dataset_path(f'{name}_arraylayout.fits.gz') return load_array_layout_from_file(layoutfile)
def test_eventseeker(): dataset = get_dataset_path("chec_r1.tio") with TargetIOEventSource(input_url=dataset) as source: seeker = EventSeeker(source) event = seeker[0] assert source._event_index == 0 assert source._event_id == 2 assert event.count == 0 assert event.r1.event_id == 2 assert (round(source._r1_samples[0, 0, 0]) == -274) event = seeker['2'] assert source._event_index == 0 assert source._event_id == 2 assert event.count == 0 assert event.r1.event_id == 2 assert (round(source._r1_samples[0, 0, 0]) == -274) event = seeker[-1] assert event.count == len(seeker) - 1 with TargetIOEventSource(input_url=dataset, max_events=3) as source: with pytest.raises(IndexError): seeker = EventSeeker(source) seeker[5]
def test_pipeline(): dataset = get_dataset_path("chec_r1.tio") reader = TargetIOEventSource(input_url=dataset, max_events=10) calibrator = CameraCalibrator(eventsource=reader) for event in reader: calibrator.calibrate(event) assert event.r0.tel.keys() == event.dl1.tel.keys()
def test_event_id(): url = get_dataset_path("chec_r1.tio") source = TargetIOEventSource(input_url=url) event_id = 2 source._get_event_by_id(event_id) assert(event_id == source._reader.fCurrentEventID) assert(round(source._r1_samples[0, 0, 0]) == -274)
def test_hessio_event_source(): filename = get_dataset_path("gamma_test.simtel.gz") with HESSIOEventSource(input_url=filename) as source: event = next(iter(source)) tels = event.dl0.tels_with_data assert tels == {38, 47}
def test_bokeh_file_viewer(): from ctapipe.tools.bokeh.file_viewer import BokehFileViewer sys.argv = ['bokeh_file_viewer'] tool = BokehFileViewer(disable_server=True) tool.run() assert tool.reader.input_url == get_dataset_path("gamma_test.simtel.gz")
def test_eventsource_override_r1(): dataset = get_dataset_path("gamma_test.simtel.gz") eventsource = HESSIOEventSource(input_url=dataset) calibrator = CameraCalibrator( eventsource=eventsource, r1_product="NullR1Calibrator" ) assert isinstance(calibrator.r1, NullR1Calibrator)
def test_factory_unknown_reader(): with pytest.raises(TraitError): dataset = get_dataset_path("gamma_test.simtel.gz") reader = EventSourceFactory.produce( product='UnknownFileReader', input_url=dataset ) assert reader is not None
def test_bokeh_file_viewer(tmpdir): from ctapipe.tools.bokeh.file_viewer import BokehFileViewer sys.argv = ["bokeh_file_viewer"] tool = BokehFileViewer(disable_server=True) assert run_tool(tool, cwd=tmpdir) == 0 assert tool.reader.input_url == get_dataset_path("gamma_test_large.simtel.gz") assert run_tool(tool, ["--help-all"]) == 0
def test_factory_from_reader(): dataset = get_dataset_path("gamma_test.simtel.gz") reader = EventSourceFactory.produce( product='SimTelEventSource', input_url=dataset ) assert reader.__class__.__name__ == "SimTelEventSource" assert reader.input_url == dataset
def test_factory_allowed_tels_from_config(): dataset = get_dataset_path("gamma_test_large.simtel.gz") config = Config({'EventSource': { 'input_url': dataset, 'allowed_tels': {1, 3} }}) reader = EventSource.from_config(config=config, parent=None) assert len(reader.allowed_tels) == 2
def test_singlemodule_r0(): url = get_dataset_path("targetmodule_r0.tio") source = TargetIOEventSource(input_url=url) event = source._get_event_by_index(0) assert (round(source._r0_samples[0, 0, 0]) == 600) assert (source._r1_samples is None) assert (event.r0.tels_with_data == {0}) assert (event.r0.tel[0].waveform[0, 0, 0] == source._r0_samples[0, 0, 0])
def test_singlemodule_r0(): url = get_dataset_path("targetmodule_r0.tio") source = TargetIOEventSource(input_url=url) event = source._get_event_by_index(0) assert(round(source._r0_samples[0, 0, 0]) == 600) assert(source._r1_samples is None) assert(event.r0.tels_with_data == {0}) assert(event.r0.tel[0].waveform[0, 0, 0] == source._r0_samples[0, 0, 0])
def test_factory_from_eventsource_override(): dataset = get_dataset_path("gamma_test.simtel.gz") eventsource = SimTelEventSource(input_url=dataset) calibrator = CameraR1CalibratorFactory.produce( eventsource=eventsource, product="NullR1Calibrator" ) assert isinstance(calibrator, NullR1Calibrator)
def test_extract_charge_resolution(tmpdir): output_path = os.path.join(str(tmpdir), "cr.h5") tool = ChargeResolutionGenerator() with pytest.raises(KeyError): tool.run([ '-f', get_dataset_path("gamma_test_large.simtel.gz"), '-o', output_path, ])
def test_eventsource_override_r1(): dataset = get_dataset_path("gamma_test.simtel.gz") eventsource = SimTelEventSource(input_url=dataset) calibrator = CameraCalibrator( eventsource=eventsource, r1_product="NullR1Calibrator" ) assert isinstance(calibrator.r1, NullR1Calibrator)
def setup(self): self.eventsource = EventSource.from_url( get_dataset_path("gamma_test_large.simtel.gz"), parent=self, ) self.calibrator = CameraCalibrator(parent=self) self.plotter = ImagePlotter(parent=self)
def test_event(): """ an example event for algorithm testing""" filename = get_dataset_path('gamma_test.simtel.gz') with HESSIOEventSource(input_url=filename) as reader: seeker = EventSeeker(reader) event = seeker['409'] yield event
def test_factory_max_events_from_config(): dataset = get_dataset_path("gamma_test_large.simtel.gz") max_events = 10 config = Config({'EventSource': { 'input_url': dataset, 'max_events': max_events, }}) reader = EventSource.from_config(config=config) assert reader.max_events == max_events
def test_subarray_property(): dataset = get_dataset_path("gamma_test_large.simtel.gz") source = HESSIOEventSource(input_url=dataset) subarray = deepcopy(source.subarray) event = next(iter(source)) subarray_event = event.inst.subarray assert subarray.tel.keys() == subarray_event.tel.keys() assert (subarray.tel[1].camera.geometry.pix_x == subarray_event.tel[1].camera.geometry.pix_x).all()
def test_from_config_default(): old_default = EventSource.input_url.default_value dataset = get_dataset_path("gamma_test_large.simtel.gz") EventSource.input_url.default_value = dataset config = Config() reader = EventSource.from_config(config=config, parent=None) assert isinstance(reader, SimTelEventSource) assert reader.input_url == dataset EventSource.input_url.default_value = old_default
def test_allowed_tels_from_config(): dataset = get_dataset_path(prod5_path) config = Config( {"EventSource": { "input_url": dataset, "allowed_tels": {1, 3} }}) reader = EventSource(config=config, parent=None) assert reader.allowed_tels == {1, 3}
def test_allowed_tels_from_config(): dataset = get_dataset_path("gamma_test_large.simtel.gz") config = Config( {"EventSource": { "input_url": dataset, "allowed_tels": {1, 3} }}) reader = EventSource.from_config(config=config, parent=None) assert len(reader.allowed_tels) == 2
def test_factory_allowed_tels_from_config(): dataset = get_dataset_path("gamma_test_large.simtel.gz") config = Config( {'EventSource': { 'input_url': dataset, 'allowed_tels': {1, 3} }}) reader = EventSource.from_config(config=config, parent=None) assert len(reader.allowed_tels) == 2
def test_geom(): dataset = get_dataset_path( "20131004_M1_05029747.003_Y_MagicCrab-W0.40+035.root") dataset = dataset.replace('_M1_', '_M*_') with MAGICEventSourceROOT(input_url=dataset) as source: event = next(source._generator()) assert event.inst.subarray.tels[1].camera.pix_x.size == 1039 assert event.inst.subarray.tels[2].camera.pix_x.size == 1039
def test_from_config_default(): old_default = EventSource.input_url.default_value dataset = get_dataset_path(prod5_path) EventSource.input_url.default_value = dataset config = Config() reader = EventSource(config=config, parent=None) assert isinstance(reader, SimTelEventSource) assert reader.input_url == dataset EventSource.input_url.default_value = old_default
def test_max_events_from_config(): dataset = get_dataset_path("gamma_test_large.simtel.gz") max_events = 10 config = Config({'EventSource': { 'input_url': dataset, 'max_events': max_events, }}) reader = EventSource.from_config(config=config) assert reader.max_events == max_events
def test_reconstruction(): """ a test of the complete fit procedure on one event including: • tailcut cleaning • hillas parametrisation • HillasPlane creation • direction fit • position fit in the end, proper units in the output are asserted """ filename = get_dataset_path("gamma_test.simtel.gz") fit = HillasReconstructor() tel_azimuth = {} tel_altitude = {} source = EventSourceFactory.produce(input_url=filename) for event in source: hillas_dict = {} for tel_id in event.dl0.tels_with_data: geom = event.inst.subarray.tel[tel_id].camera tel_azimuth[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad tel_altitude[tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad pmt_signal = event.r0.tel[tel_id].image[0] mask = tailcuts_clean(geom, pmt_signal, picture_thresh=10., boundary_thresh=5.) pmt_signal[mask == 0] = 0 try: moments = hillas_parameters(geom, pmt_signal) hillas_dict[tel_id] = moments except HillasParameterizationError as e: print(e) continue if len(hillas_dict) < 2: continue fit_result = fit.predict(hillas_dict, event.inst, tel_azimuth, tel_altitude) print(fit_result) fit_result.alt.to(u.deg) fit_result.az.to(u.deg) fit_result.core_x.to(u.m) assert fit_result.is_valid return
def test_eventseeker(): dataset = get_dataset_path("gamma_test.simtel.gz") kwargs = dict(config=None, tool=None, input_url=dataset) with SimTelEventSource(**kwargs) as reader: seeker = EventSeeker(reader=reader) event = seeker[1] assert event.r0.tels_with_data == {11, 21, 24, 26, 61, 63, 118, 119} event = seeker[0] assert event.r0.tels_with_data == {38, 47} event = seeker['409'] assert event.r0.event_id == 409 assert event.r0.tels_with_data == {11, 21, 24, 26, 61, 63, 118, 119} tel_list = [{38, 47}, {11, 21, 24, 26, 61, 63, 118, 119}] events = seeker[0:2] events_tels = [e.r0.tels_with_data for e in events] assert events_tels == tel_list events = seeker[[0, 1]] events_tels = [e.r0.tels_with_data for e in events] assert events_tels == tel_list events = seeker[['408', '409']] events_tels = [e.r0.tels_with_data for e in events] assert events_tels == tel_list assert len(seeker) == 9 with pytest.raises(IndexError): event = seeker[200] assert event is not None with pytest.raises(ValueError): event = seeker['t'] assert event is not None with pytest.raises(TypeError): event = seeker[dict()] assert event is not None with SimTelEventSource(**kwargs, max_events=5) as reader: seeker = EventSeeker(reader=reader) with pytest.raises(IndexError): event = seeker[5] assert event is not None class StreamFileReader(SimTelEventSource): def is_stream(self): return True with StreamFileReader(**kwargs) as reader: with pytest.raises(IOError): seeker = EventSeeker(reader=reader) assert seeker is not None
def example_subarray(): """ Subarray corresponding to the example event """ filename = get_dataset_path("gamma_test_large.simtel.gz") print("******************** LOAD TEST EVENT ***********************") with SimTelEventSource(input_url=filename) as reader: return reader.subarray
def test_chec_r1(): url = get_dataset_path("chec_r1.tio") source = TargetIOEventSource(input_url=url) event = source._get_event_by_index(0) assert(source._r0_samples is None) assert(source._r1_samples.shape[1] == 2048) assert(round(source._r1_samples[0, 0, 0]) == -274) assert(event.r0.tels_with_data == {0}) assert(event.r0.tel[0].waveform is None) assert(event.r1.tel[0].waveform[0, 0, 0] == source._r1_samples[0, 0, 0])
def test_jsonToFits(): backup = sys.argv full_config_name = get_dataset_path('config.json') sys.argv = ['test_json_2_fits.py', '--config_f=' + full_config_name] app = MyApp() app.initialize() app.start() tmp = tempfile.NamedTemporaryFile() app.json_to_fits(str(tmp.name)) sys.argv = backup
def test_len(): dataset = get_dataset_path("chec_r1.tio") with TargetIOEventSource(input_url=dataset) as source: count = 0 for _ in source: count += 1 assert count == len(source) with TargetIOEventSource(input_url=dataset, max_events=3) as reader: assert len(reader) == 3
def test_max_events_from_config(): dataset = get_dataset_path(prod5_path) max_events = 10 config = Config( {"EventSource": { "input_url": dataset, "max_events": max_events }}) reader = EventSource(config=config) assert reader.max_events == max_events
def test_max_events_from_config(): dataset = get_dataset_path("gamma_test_large.simtel.gz") max_events = 10 config = Config( {"EventSource": { "input_url": dataset, "max_events": max_events }}) reader = EventSource.from_config(config=config) assert reader.max_events == max_events
def test_dump_instrument(tmpdir): tmpdir.chdir() tool = DumpInstrumentTool( infile=get_dataset_path("gamma_test_large.simtel.gz"), ) tool.run(argv=[]) print(tmpdir.listdir()) assert tmpdir.join('FlashCam.camgeom.fits.gz').exists()
def test_dump_triggers(tmpdir): outfile = tmpdir.join("triggers.fits") tool = DumpTriggersTool( infile=get_dataset_path("gamma_test_large.simtel.gz"), outfile=str(outfile)) tool.run(argv=[]) assert outfile.exists()
def test_chec_r1(): url = get_dataset_path("chec_r1.tio") source = TargetIOEventSource(input_url=url) event = source._get_event_by_index(0) assert (source._r0_samples is None) assert (source._r1_samples.shape[1] == 2048) assert (round(source._r1_samples[0, 0, 0]) == -274) assert (event.r0.tels_with_data == {0}) assert (event.r0.tel[0].waveform is None) assert (event.r1.tel[0].waveform[0, 0, 0] == source._r1_samples[0, 0, 0])
def validate(self, obj, value): if isinstance(value, bytes): value = os.fsdecode(value) if value is None or value is Undefined: if self.allow_none: return value else: self.error(obj, value) if not isinstance(value, (str, pathlib.Path)): return self.error(obj, value) # expand any environment variables in the path: value = os.path.expandvars(value) if isinstance(value, str): if value == "": return self.error(obj, value) try: url = urlparse(value) except ValueError: return self.error(obj, value) if url.scheme in ("http", "https"): # here to avoid circular import, since every module imports # from ctapipe.core from ctapipe.utils.download import download_cached value = download_cached(value, progress=True) elif url.scheme == "dataset": # here to avoid circular import, since every module imports # from ctapipe.core from ctapipe.utils import get_dataset_path value = get_dataset_path(value.partition("dataset://")[2]) elif url.scheme in ("", "file"): value = pathlib.Path(url.netloc, url.path) else: return self.error(obj, value) value = value.absolute() exists = value.exists() if self.exists is not None: if exists != self.exists: raise TraitError('Path "{}" {} exist'.format( value, "does not" if self.exists else "must not")) if exists: if not self.directory_ok and value.is_dir(): raise TraitError(f'Path "{value}" must not be a directory') if not self.file_ok and value.is_file(): raise TraitError(f'Path "{value}" must not be a file') return value
def init(self): self.log.debug('%self.filename' % "--- SimTelArrayReader init {}---") try: in_file = get_dataset_path(self.filename) self.source = event_source(in_file, max_events=3) self.log.debug('%s successfully opened %s', self.filename, self.source) except Exception: self.log.error('could not open ' + in_file) return False return True
def testCtaContainerExtractor(self): file_path = get_dataset_path("gamma_test_large.simtel.gz") print(file_path) start = time.time() output_json = metadataextractorCtaContainer.extract(file_path) print("output_json extractor" + str(output_json)) self.assertIn( '{"is_simulation": true, "pointing_direction_alt": 1.2217304706573486, "pointing_direction_az": 0.0, "is_diffuse": true, "particle_type": 0, "run_id": 7514, "time": 1467926303, "simtel_version": 1462392225}', output_json) end = time.time() print("time to parse files is {:f} s".format(end - start))
def test_bokeh_file_viewer(): from ctapipe.tools.bokeh.file_viewer import BokehFileViewer sys.argv = ['bokeh_file_viewer'] tool = BokehFileViewer(disable_server=True) tool.run() assert tool.reader.input_url == get_dataset_path("gamma_test_large.simtel.gz") with pytest.raises(SystemExit): tool.run(['--help-all'])
def setup(self): kwargs = dict(config=self.config, tool=self) self.eventsource = EventSourceFactory.produce( input_url=get_dataset_path("gamma_test.simtel.gz"), **kwargs ) self.calibrator = CameraCalibrator( eventsource=self.eventsource, **kwargs ) self.plotter = ImagePlotter(**kwargs)
def test_that_event_is_not_modified_after_loop(): dataset = get_dataset_path("chec_r1.tio") with TargetIOEventSource(input_url=dataset, max_events=2) as source: for event in source: last_event = copy.deepcopy(event) # now `event` should be identical with the deepcopy of itself from # inside the loop. # Unfortunately this does not work: # assert last_event == event # So for the moment we just compare event ids assert event.r0.event_id == last_event.r0.event_id
def test_dump_triggers(tmpdir): sys.argv = ['dump_triggers'] outfile = tmpdir.join("triggers.fits") tool = DumpTriggersTool( infile=get_dataset_path("gamma_test_large.simtel.gz"), outfile=str(outfile) ) tool.run(argv=[]) assert outfile.exists()
def test_dump_instrument(tmpdir): sys.argv = ['dump_instrument'] tmpdir.chdir() tool = DumpInstrumentTool( infile=get_dataset_path("gamma_test_large.simtel.gz"), ) tool.run(argv=[]) print(tmpdir.listdir()) assert tmpdir.join('FlashCam.camgeom.fits.gz').exists()
def setup(self): self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]" kwargs = dict(config=self.config, tool=self) default_url = get_dataset_path("gamma_test.simtel.gz") EventSourceFactory.input_url.default_value = default_url self.reader = EventSourceFactory.produce(**kwargs) self.seeker = EventSeeker(self.reader, **kwargs) self.extractor = ChargeExtractorFactory.produce(**kwargs) self.cleaner = WaveformCleanerFactory.produce(**kwargs) self.r1 = CameraR1CalibratorFactory.produce( eventsource=self.reader, **kwargs ) self.dl0 = CameraDL0Reducer(**kwargs) self.dl1 = CameraDL1Calibrator( extractor=self.extractor, cleaner=self.cleaner, **kwargs ) self.viewer = BokehEventViewer(**kwargs) # Setup widgets self.viewer.create() self.viewer.enable_automatic_index_increment() self.create_previous_event_widget() self.create_next_event_widget() self.create_event_index_widget() self.create_goto_event_index_widget() self.create_event_id_widget() self.create_goto_event_id_widget() self.create_telid_widget() self.create_channel_widget() self.create_dl1_widgets() self.update_dl1_widget_values() # Setup layout self.layout = layout([ [self.viewer.layout], [ self.w_previous_event, self.w_next_event, self.w_goto_event_index, self.w_goto_event_id ], [self.w_event_index, self.w_event_id], [self.w_telid, self.w_channel], [self.wb_extractor] ])
def test_reconstruction(): """ a test of the complete fit procedure on one event including: • tailcut cleaning • hillas parametrisation • HillasPlane creation • direction fit • position fit in the end, proper units in the output are asserted """ filename = get_dataset_path("gamma_test.simtel.gz") fit = HillasReconstructor() tel_azimuth = {} tel_altitude = {} source = EventSourceFactory.produce(input_url=filename) for event in source: hillas_dict = {} for tel_id in event.dl0.tels_with_data: geom = event.inst.subarray.tel[tel_id].camera tel_azimuth[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad tel_altitude[tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad pmt_signal = event.r0.tel[tel_id].image[0] mask = tailcuts_clean(geom, pmt_signal, picture_thresh=10., boundary_thresh=5.) pmt_signal[mask == 0] = 0 try: moments = hillas_parameters(geom, pmt_signal) hillas_dict[tel_id] = moments except HillasParameterizationError as e: print(e) continue if len(hillas_dict) < 2: continue fit_result = fit.predict(hillas_dict, event.inst, tel_azimuth, tel_altitude) print(fit_result) fit_result.alt.to(u.deg) fit_result.az.to(u.deg) fit_result.core_x.to(u.m) assert fit_result.is_valid
def _global_example_event(): """ helper to get a single event from a MC file. Don't use this fixture directly, rather use `test_event` """ filename = get_dataset_path('gamma_test_large.simtel.gz') print("******************** LOAD TEST EVENT ***********************") with SimTelEventSource(input_url=filename) as reader: event = next(iter(reader)) return event
def test_loop(): dataset = get_dataset_path("chec_r1.tio") with TargetIOEventSource(input_url=dataset) as source: count = 0 for event in source: assert event.r0.tels_with_data == {0} assert event.count == count count += 1 for event in source: # Check generator has restarted from beginning assert event.count == 0 break
def test_eventplotter(): dataset = get_dataset_path("gamma_test_large.simtel.gz") with event_source(dataset, max_events=1) as source: event = next(iter(source)) telid = list(event.r0.tels_with_data)[0] data = event.r0.tel[telid].waveform[0] plotter = CameraPlotter(event) camera = plotter.draw_camera(telid, data[:, 0]) assert camera is not None np.testing.assert_array_equal(camera.image, data[:, 0]) plotter.draw_camera_pixel_ids(telid, [0, 1, 2]) waveform = plotter.draw_waveform(data[0, :]) assert waveform is not None np.testing.assert_array_equal(waveform.get_ydata(), data[0, :]) line = plotter.draw_waveform_positionline(0) assert line is not None np.testing.assert_array_equal(line.get_xdata(), [0, 0])