def __init__(self, input_url): self._file = {} self._events = {} self._events_table = {} self._camera_config = {} paths = [ input_url, ] # test how many files are there if '000.fits.fz' in input_url: i = 0 while True: input_url = input_url.replace( str(i).zfill(3) + '.fits.fz', str(i + 1).zfill(3) + '.fits.fz') if exists(input_url): paths.append(input_url) # keep track of all input files Provenance().add_input_file(input_url, role='dl0.sub.evt') i = i + 1 else: break # open the files and get the first fits Tables from protozfits import File for path in paths: self._file[path] = File(path) self._events_table[path] = File(path).Events try: self._events[path] = next(self._file[path].Events) self._camera_config[path] = next(self._file[path].CameraConfig) except StopIteration: pass # verify that the configuration_id of all files are the same # in the CameraConfig table for path in paths: assert (self._camera_config[path].configuration_id == self._camera_config[paths[0]].configuration_id) # keep the cameraConfig of the first file self.camera_config = self._camera_config[paths[0]]
def load_r1(self, fits_file, fits_mode=1, test=False): if not fits_file: self.log.error("No FITS file provided") return False if not os.path.isfile(fits_file): self.log.error("Bad file provided") return False self.log.info("Loading FITS file %s in mode %d", fits_file, fits_mode) if fits_mode == 0: try: f = fits.open(fits_file) except: self.log.error("Invalid FITS file provided") return False sc = self.__load_schema_from_astropy(f, test) f.close() return sc if fits_mode == 1: try: f = File(fits_file) except OSError: self.log.error("Invalid FITS file provided") return False sc = self.__load_schema_from_protozfits(f, test) f.close() return sc
def __init__(self, config=None, tool=None, **kwargs): super().__init__(config=config, tool=tool, **kwargs) from protozfits import File self.file = File(self.input_url) # TODO: Correct pixel ordering self._tel_desc = TelescopeDescription.from_name(optics_name='SST-1M', camera_name='DigiCam')
def test_File_getitem_with_range(): f = File(example_file_path) interesting_event_ids = range(9, 1, -2) expected_event_numbers = [ FIRST_EVENT_NUMBER + i for i in interesting_event_ids ] for i, event in enumerate(f.Events[interesting_event_ids]): assert event.eventNumber == expected_event_numbers[i]
def test_File_getitem_with_slice(): f = File(example_file_path) expected_event_numbers = [ FIRST_EVENT_NUMBER + 1, FIRST_EVENT_NUMBER + 2, ] for i, event in enumerate(f.Events[1:3]): assert event.eventNumber == expected_event_numbers[i]
def test_File_getitem_with_iterable(): f = File(example_file_path) expected_event_numbers = [ FIRST_EVENT_NUMBER + 3, FIRST_EVENT_NUMBER + 7, FIRST_EVENT_NUMBER + 1, ] for i, event in enumerate(f.Events[[3, 7, 1]]): assert event.eventNumber == expected_event_numbers[i]
def test_can_iterate_over_events_and_run_header(): with File(example_file_path) as f: camera_config = next(f.CameraConfig) assert (camera_config.expected_pixels_id == np.arange(14)).all() for i, event in enumerate(f.Events): assert event.event_id == i + 1 assert event.waveform.shape == (1120, ) assert event.pixel_status.shape == (14, ) assert event.lstcam.first_capacitor_id.shape == (16, ) assert event.lstcam.counters.shape == (44, )
def __init__(self, file_list): self._file = {} self._events = {} self._events_table = {} self._camera_config = {} self.camera_config = None paths = [] for file_name in file_list: paths.append(file_name) Provenance().add_input_file(file_name, role='r0.sub.evt') # open the files and get the first fits Tables from protozfits import File for path in paths: try: self._file[path] = File(path) self._events_table[path] = File(path).Events self._events[path] = next(self._file[path].Events) # verify where the CameraConfig is present if 'CameraConfig' in self._file[path].__dict__.keys(): self._camera_config[path] = next( self._file[path].CameraConfig) # for the moment it takes the first CameraConfig it finds (to be changed) if (self.camera_config is None): self.camera_config = self._camera_config[path] except StopIteration: pass # verify that somewhere the CameraConfing is present assert self.camera_config
def __init__(self, paths): """ Create a new MultiFiles object from an iterable of paths Parameters ---------- paths: Iterable[string|Path] The input paths """ paths = list(paths) if len(paths) == 0: raise ValueError('`paths` must not be empty') self._file = {} self._events = {} self._events_table = {} self._camera_config = {} for path in paths: Provenance().add_input_file(path, role='r0.sub.evt') try: self._file[path] = File(str(path)) self._events_table[path] = self._file[path].Events self._events[path] = next(self._file[path].Events) if hasattr(self._file[path], 'CameraConfig'): self._camera_config[path] = next(self._file[path].CameraConfig) else: warnings.warn(f'No CameraConfig found in {path}') except StopIteration: pass run_ids = { config.configuration_id for config in self._camera_config.values() } if len(run_ids) > 1: raise IOError(f'Found multiple run_ids: {run_ids}') # verify that we found a CameraConfig if len(self._camera_config) == 0: raise IOError(f"No CameraConfig was found in any of the input files: {paths}") else: self.camera_config = next(iter(self._camera_config.values()))
def test_can_open_and_get_an_event_from_all_test_resources(): print() for path in all_test_resources: with File(path) as f: event = next(f.Events) print(path, len(str(event)))
def __init__(self, config=None, tool=None, **kwargs): super().__init__(config=config, tool=tool, **kwargs) from protozfits import File self.file = File(self.input_url) self.header = next(self.file.RunHeader)
def zfits_event_source(url, camera=camera.DigiCam, max_events=None, allowed_tels=None, event_id=None, disable_bar=False): """A generator that streams data from an ZFITs data file Parameters ---------- url : str path to file to open max_events : int, optional maximum number of events to read allowed_tels : list[int] select only a subset of telescope, if None, all are read. This can be used for example emulate the final CTA data format, where there would be 1 telescope per file (whereas in current monte-carlo, they are all interleaved into one file) camera : digicampipe.instrument.Camera(), default DigiCam event_id: int Event id to start at. If the exact event ID does not exists it will return the closest past event. If the event ID is out of the range of the file it will raise an IndexError disable_bar: If set to true, the progress bar is not shown. """ data = DataContainer() with File(url) as file: loaded_telescopes = [] n_events_in_file = len(file.Events) events = file.Events index_of_event = 0 if event_id is not None: index_of_event = _binary_search(file, event_id) first_event_id = file.Events[0].eventNumber last_event_id = file.Events[n_events_in_file - 1].eventNumber if not first_event_id <= event_id <= last_event_id: raise IndexError('Cannot find event ID {} in File {}\n' 'First event ID : {}\n' 'Last event ID : {}'.format( event_id, url, first_event_id, last_event_id)) events = events[max(index_of_event, 0):] n_steps = n_events_in_file if max_events is None else max_events for event_counter, event in tqdm(enumerate(events), desc='Events', leave=True, initial=index_of_event, disable=disable_bar, total=n_steps): if max_events is not None and event_counter > max_events: break data.r0.event_id = event_counter data.r0.tels_with_data = [ event.telescopeID, ] # remove forbidden telescopes if allowed_tels: data.r0.tels_with_data = [ list(filter(lambda x: x in data.r0.tels_with_data, sublist)) for sublist in allowed_tels ] _sort_ids = None for tel_id in data.r0.tels_with_data: pixel_ids = event.hiGain.waveforms.pixelsIndices n_pixels = len(pixel_ids) if _sort_ids is None: _sort_ids = np.argsort(pixel_ids) samples = event.hiGain.waveforms.samples.reshape(n_pixels, -1) try: unsorted_baseline = event.hiGain.waveforms.baselines except AttributeError: warnings.warn( ("Could not read `hiGain.waveforms.baselines`" "for event:{}\n" "of file:{}\n".format(event_counter, url))) return np.ones(n_pixels) * np.nan if tel_id not in loaded_telescopes: data.inst.num_channels[tel_id] = event.num_gains data.inst.geom[tel_id] = camera.geometry data.inst.cluster_matrix_7[tel_id] = \ camera.cluster_7_matrix data.inst.cluster_matrix_19[tel_id] = \ camera.cluster_19_matrix data.inst.patch_matrix[tel_id] = camera.patch_matrix data.inst.num_pixels[tel_id] = samples.shape[0] data.inst.num_samples[tel_id] = samples.shape[1] loaded_telescopes.append(tel_id) r0 = data.r0.tel[tel_id] r0.camera_event_number = event.eventNumber r0.pixel_flags = event.pixels_flags[_sort_ids] r0.local_camera_clock = (np.int64(event.local_time_sec * 1E9) + np.int64(event.local_time_nanosec)) r0.gps_time = (np.int64(event.trig.timeSec * 1E9) + np.int64(event.trig.timeNanoSec)) r0.camera_event_type = event.event_type r0.array_event_type = event.eventType r0.adc_samples = samples[_sort_ids] if len(event.trigger_input_traces) > 0: r0.trigger_input_traces = _prepare_trigger_input( event.trigger_input_traces) else: warnings.warn( 'trigger_input_traces does not exist: --> nan') r0.trigger_input_traces = np.zeros( (432, data.inst.num_samples[tel_id])) * np.nan if len(event.trigger_output_patch7) > 0: r0.trigger_output_patch7 = _prepare_trigger_output( event.trigger_output_patch7) else: warnings.warn( 'trigger_output_patch7 does not exist: --> nan') r0.trigger_output_patch7 = np.zeros( (432, data.inst.num_samples[tel_id])) * np.nan if len(event.trigger_output_patch19) > 0: r0.trigger_output_patch19 = _prepare_trigger_output( event.trigger_output_patch19) else: warnings.warn( 'trigger_output_patch19 does not exist: --> nan') r0.trigger_output_patch19 = np.zeros( (432, data.inst.num_samples[tel_id])) * np.nan r0.digicam_baseline = unsorted_baseline[_sort_ids] / 16 yield data
def count_number_events(file_list): return sum(len(File(filename).Events) for filename in file_list)
def test_File_getitem_with_integer(): f = File(example_file_path) event = f.Events[0] assert event.eventNumber == FIRST_EVENT_NUMBER
def func(): for _, i in zip(File(example_file_path).Events, range(100)): pass assert i == 99