def test_provenence_contextmanager(): prov = Provenance() with prov.activity("myactivity"): assert 'myactivity' in prov.active_activity_names assert 'myactivity' in prov.finished_activity_names assert 'myactivity' not in prov.active_activity_names
def save(self): """ Save the figure to the path defined by the `output_path` trait """ self._finish() output_dir = os.path.dirname(self.output_path) if not os.path.exists(output_dir): print(f"Creating directory: {output_dir}") os.makedirs(output_dir) self.fig.savefig(self.output_path, bbox_inches='tight') print(f"Figure saved to: {self.output_path}") Provenance().add_output_file(self.output_path) plt.close(self.fig)
def finish(self): """ finish up and write out results (called automatically after `start()`) """ # write out the final table if self.outfile.endswith('fits') or self.outfile.endswith('fits.gz'): self.events.write(self.outfile, overwrite=self.overwrite) elif self.outfile.endswith('h5'): self.events.write(self.outfile, path='/events', overwrite=self.overwrite) else: self.events.write(self.outfile) Provenance().add_output_file(self.outfile) self.log.info("Table written to '{}'".format(self.outfile)) self.log.info('\n %s', self.events)
def setup(self): self.filename_dl3 = dl2_to_dl3_filename(self.input_dl2, compress=self.gzip) self.provenance_log = self.output_dl3_path / (self.name + ".provenance.log") Provenance().add_input_file(self.input_dl2) self.event_sel = EventSelector(parent=self) self.cuts = DL3Cuts(parent=self) self.output_file = self.output_dl3_path.absolute() / self.filename_dl3 if self.output_file.exists(): if self.overwrite: self.log.warning(f"Overwriting {self.output_file}") self.output_file.unlink() else: raise ToolConfigurationError( f"Output file {self.output_file} already exists," " use --overwrite to overwrite") if not (self.source_ra or self.source_dec): self.source_pos = SkyCoord.from_name(self.source_name) elif bool(self.source_ra) != bool(self.source_dec): raise ToolConfigurationError( "Either provide both RA and DEC values for the source or none") else: self.source_pos = SkyCoord(ra=self.source_ra, dec=self.source_dec) self.log.debug(f"Output DL3 file: {self.output_file}") try: with fits.open(self.input_irf) as hdul: self.use_energy_dependent_gh_cuts = ( "GH_CUT" not in hdul["EFFECTIVE AREA"].header) except: raise ToolConfigurationError( f"{self.input_irf} does not have EFFECTIVE AREA HDU, " " to check for global cut information in the Header value") if self.source_dep: with fits.open(self.input_irf) as hdul: self.use_energy_dependent_alpha_cuts = ( "AL_CUT" not in hdul["EFFECTIVE AREA"].header)
def setup(self): self.list_files = sorted(self.input_dl3_dir.glob(self.file_pattern)) if len(self.list_files) == 0: raise ToolConfigurationError( f"No files found with pattern {self.file_pattern} in {self.input_dl3_dir}" ) for f in self.list_files: Provenance().add_input_file(f) if not self.output_index_path: self.output_index_path = self.input_dl3_dir self.hdu_index_file = self.output_index_path / self.hdu_index_filename self.obs_index_file = self.output_index_path / self.obs_index_filename self.provenance_log = self.output_index_path / (self.name + ".provenance.log") if self.hdu_index_file.exists(): if self.overwrite: self.log.warning(f"Overwriting {self.hdu_index_file}") self.hdu_index_file.unlink() else: raise ToolConfigurationError( f"Output file {self.hdu_index_file} already exists," "use --overwrite to overwrite" ) if self.obs_index_file.exists(): if self.overwrite: self.log.warning(f"Overwriting {self.obs_index_file}") self.obs_index_file.unlink() else: raise ToolConfigurationError( f"Output file {self.obs_index_file} already exists," " use --overwrite to overwrite" ) self.log.debug("HDU Index file: %s", self.hdu_index_file) self.log.debug("OBS Index file: %s", self.obs_index_file)
def write_camera_geometries(self): cam_types = get_camera_types(self.inst.subarray) self.inst.subarray.info(printer=self.log.info) for cam_name in cam_types: ext, args = self._get_file_format_info(self.format, 'CAMGEOM', cam_name) self.log.debug("writing {}".format(cam_name)) tel_id = cam_types[cam_name].pop() geom = self.inst.subarray.tel[tel_id].camera table = geom.to_table() table.meta['SOURCE'] = self.infile filename = "{}.camgeom.{}".format(cam_name, ext) try: table.write(filename, **args) Provenance().add_output_file(filename, 'dl0.tel.svc.camera') except IOError as err: self.log.warn("couldn't write camera definition '%s' because: " "%s", filename, err)
def write_camera_geometries(self): cam_types = get_camera_types(self.subarray) self.subarray.info(printer=self.log.info) for cam_name in cam_types: ext, args = self._get_file_format_info(self.format, "CAMGEOM", cam_name) self.log.debug(f"writing {cam_name}") tel_id = cam_types[cam_name].pop() geom = self.subarray.tel[tel_id].camera.geometry table = geom.to_table() table.meta["SOURCE"] = str(self.infile) filename = f"{cam_name}.camgeom.{ext}" try: table.write(filename, **args) Provenance().add_output_file(filename, "dl0.tel.svc.camera") except IOError as err: self.log.warning( "couldn't write camera definition '%s' because: %s", filename, err)
def finish(self): """ finish up and write out results (called automatically after `start()`) """ # write out the final table try: if '.fits' in self.outfile.suffixes: self.events.write(self.outfile, overwrite=self.overwrite) elif self.outfile.suffix in ('.hdf5', '.h5', '.hdf'): self.events.write(self.outfile, path='/events', overwrite=self.overwrite) else: self.events.write(self.outfile) Provenance().add_output_file(self.outfile) except IOError as err: self.log.warning("Couldn't write output (%s)", err) self.log.info('\n %s', self.events)
def finish(self): """ finish up and write out results (called automatically after `start()`) """ # write out the final table try: if self.outfile.endswith('fits') or self.outfile.endswith( 'fits.gz'): self.events.write(self.outfile, overwrite=self.overwrite) elif self.outfile.endswith('h5'): self.events.write(self.outfile, path='/events', overwrite=self.overwrite) else: self.events.write(self.outfile) Provenance().add_output_file(self.outfile) except IOError as err: self.log.warn("Couldn't write output (%s)", err) self.log.info('\n %s', self.events)
def __init__(self, file_list): self._file = {} self._events = {} self._events_table = {} self._camera_config = {} self.camera_config = None paths = [] for file_name in file_list: paths.append(file_name) Provenance().add_input_file(file_name, role='r0.sub.evt') # open the files and get the first fits Tables from protozfits import File for path in paths: try: self._file[path] = File(path) self._events_table[path] = File(path).Events self._events[path] = next(self._file[path].Events) # verify where the CameraConfig is present if 'CameraConfig' in self._file[path].__dict__.keys(): self._camera_config[path] = next( self._file[path].CameraConfig) # for the moment it takes the first CameraConfig it finds (to be changed) if (self.camera_config is None): self.camera_config = self._camera_config[path] except StopIteration: pass # verify that somewhere the CameraConfing is present assert self.camera_config
def hessio_get_list_event_ids(url, max_events=None): """ Faster method to get a list of all the event ids in the hessio file. This list can also be used to find out the number of events that exist in the file. Parameters ---------- url : str path to file to open max_events : int, optional maximum number of events to read Returns ------- event_id_list : list[num_events] A list with all the event ids that are in the file. """ logger.warning("This method is slow. Need to find faster method.") try: with open_hessio(url) as pyhessio_file: Provenance().add_input_file(url, role='r0.sub.evt') counter = 0 event_id_list = [] eventstream = pyhessio_file.move_to_next_event() for event_id in eventstream: event_id_list.append(event_id) counter += 1 if max_events and counter >= max_events: pyhessio_file.close_file() break return event_id_list except HessioError: raise RuntimeError( "hessio_event_source failed to open '{}'".format(url))
def finish(self): Provenance().add_output_file(self.outfile, role='dl1.tel.evt.muon') self.writer.close()
def finish(self): Provenance().add_output_file(self.hdu_index_file) Provenance().add_output_file(self.obs_index_file)
def finish(self): fits.HDUList(self.hdus).writeto(self.output_irf_file, overwrite=self.overwrite) Provenance().add_output_file(self.output_irf_file)
def test_Provenance(): prov = Provenance() prov.start_activity("test1") prov.add_input_file("input.txt") prov.add_output_file("output.txt") prov.start_activity("test2") prov.add_input_file("input_a.txt") prov.add_input_file("input_b.txt") prov.finish_activity("test2") prov.finish_activity("test1") assert set(prov.finished_activity_names) == {"test2", "test1"} return prov
def load_camera_geometry(version=4): ''' Load camera geometry from bundled resources of this repo ''' f = resource_filename('ctapipe_io_lst', f'resources/LSTCam-{version:03d}.camgeom.fits.gz') Provenance().add_input_file(f, role="CameraGeometry") return CameraGeometry.from_table(f)
def finish(self): Provenance().add_output_file( self.output_file, role='mon.tel.pedestal' ) self.writer.close()
def finish(self): Provenance().add_output_file(self.output, role="muon_efficiency_parameters") self.writer.close()
def __init__(self, **kwargs): """ Constructor Parameters ---------- n_gains = number of gains expected in input file baseline = baseline to be subtracted at r1 level (not used for the moment) multi_streams = enable the reading of input files from all streams config: traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. Set to None if no configuration to pass.\ kwargs: dict Additional parameters to be passed. NOTE: The file mask of the data to read can be passed with the 'input_url' parameter. """ # EventSource can not handle file wild cards as input_url # To overcome this we substitute the input_url with first file matching # the specified file mask (copied from MAGICEventSourceROOT). super().__init__(**kwargs) if self.multi_streams: # test how many streams are there: # file name must be [stream name]Run[all the rest] # All the files with the same [all the rest] are opened if '/' in self.input_url: dir, name = self.input_url.rsplit('/', 1) else: dir = getcwd() name = self.input_url if 'Run' in name: stream, run = name.split('Run', 1) else: run = name ls = listdir(dir) self.file_list = [] for file_name in ls: if run in file_name: full_name = dir + '/' + file_name self.file_list.append(full_name) Provenance().add_input_file(full_name, role='dl0.sub.evt') else: self.file_list = [self.input_url] self.multi_file = MultiFiles(self.file_list) self.camera_config = self.multi_file.camera_config self.log.info( "Read {} input files".format( self.multi_file.num_inputs() ) )
def hessio_event_source(url, camera_geometry, camera, max_events=None, allowed_tels=None, requested_event=None, use_event_id=False): """A generator that streams data from an EventIO/HESSIO MC data file (e.g. a standard CTA data file.) Parameters ---------- url : str path to file to open max_events : int, optional maximum number of events to read allowed_tels : list[int] select only a subset of telescope, if None, all are read. This can be used for example emulate the final CTA data format, where there would be 1 telescope per file (whereas in current monte-carlo, they are all interleaved into one file) requested_event : int Seek to a paricular event index use_event_id : bool If True ,'requested_event' now seeks for a particular event id instead of index """ with open_hessio(url) as pyhessio_file: # the container is initialized once, and data is replaced within # it after each yield Provenance().add_input_file(url, role='dl0.sub.evt') counter = 0 eventstream = pyhessio_file.move_to_next_event() if allowed_tels is not None: allowed_tels = set(allowed_tels) data = DataContainer() data.meta['origin'] = "hessio" # some hessio_event_source specific parameters data.meta['input'] = url data.meta['max_events'] = max_events for event_id in eventstream: # Seek to requested event if requested_event is not None: current = counter if use_event_id: current = event_id if not current == requested_event: counter += 1 continue data.r0.run_id = pyhessio_file.get_run_number() data.r0.event_id = event_id data.r0.tels_with_data = list(pyhessio_file.get_teldata_list()) data.r1.run_id = pyhessio_file.get_run_number() data.r1.event_id = event_id data.r1.tels_with_data = list(pyhessio_file.get_teldata_list()) data.dl0.run_id = pyhessio_file.get_run_number() data.dl0.event_id = event_id data.dl0.tels_with_data = list(pyhessio_file.get_teldata_list()) # handle telescope filtering by taking the intersection of # tels_with_data and allowed_tels if allowed_tels is not None: selected = data.r0.tels_with_data & allowed_tels if len(selected) == 0: continue # skip event data.r0.tels_with_data = selected data.r1.tels_with_data = selected data.dl0.tels_with_data = selected data.trig.tels_with_trigger \ = pyhessio_file.get_central_event_teltrg_list() time_s, time_ns = pyhessio_file.get_central_event_gps_time() data.trig.gps_time = Time(time_s * u.s, time_ns * u.ns, format='unix', scale='utc') data.mc.energy = pyhessio_file.get_mc_shower_energy() * u.TeV data.mc.alt = Angle(pyhessio_file.get_mc_shower_altitude(), u.rad) data.mc.az = Angle(pyhessio_file.get_mc_shower_azimuth(), u.rad) data.mc.core_x = pyhessio_file.get_mc_event_xcore() * u.m data.mc.core_y = pyhessio_file.get_mc_event_ycore() * u.m first_int = pyhessio_file.get_mc_shower_h_first_int() * u.m data.mc.h_first_int = first_int # mc run header data data.mcheader.run_array_direction = \ pyhessio_file.get_mc_run_array_direction() data.count = counter # this should be done in a nicer way to not re-allocate the # data each time (right now it's just deleted and garbage # collected) data.r0.tel.clear() data.r1.tel.clear() data.dl0.tel.clear() data.dl1.tel.clear() data.mc.tel.clear() # clear the previous telescopes _fill_instrument_info(data, pyhessio_file, camera_geometry, camera) for tel_id in data.r0.tels_with_data: data.mc.mc_event_offset_fov = \ pyhessio_file.get_mc_event_offset_fov() data.mc.tel[tel_id].dc_to_pe \ = pyhessio_file.get_calibration(tel_id) data.mc.tel[tel_id].pedestal \ = pyhessio_file.get_pedestal(tel_id) data.r0.tel[tel_id].adc_samples = \ pyhessio_file.get_adc_sample(tel_id) data.r0.tel[tel_id].adc_samples = \ data.r0.tel[tel_id].adc_samples[0] data.r0.tel[tel_id].local_camera_clock = 0 if data.r0.tel[tel_id].adc_samples.size == 0: # To handle ASTRI and dst files data.r0.tel[tel_id].adc_samples = \ pyhessio_file.get_adc_sum(tel_id)[..., None] data.r0.tel[tel_id].adc_sums = \ pyhessio_file.get_adc_sum(tel_id) data.mc.tel[tel_id].reference_pulse_shape = \ pyhessio_file.get_ref_shapes(tel_id) nsamples = pyhessio_file.get_event_num_samples(tel_id) if nsamples <= 0: nsamples = 1 data.r0.tel[tel_id].num_samples = nsamples # load the data per telescope/pixel hessio_mc_npe = pyhessio_file.get_mc_number_photon_electron data.mc.tel[tel_id].photo_electron_image \ = hessio_mc_npe(telescope_id=tel_id) data.mc.tel[tel_id].meta['refstep'] = \ pyhessio_file.get_ref_step(tel_id) data.mc.tel[tel_id].time_slice = \ pyhessio_file.get_time_slice(tel_id) data.mc.tel[tel_id].azimuth_raw = \ pyhessio_file.get_azimuth_raw(tel_id) data.mc.tel[tel_id].altitude_raw = \ pyhessio_file.get_altitude_raw(tel_id) data.mc.tel[tel_id].azimuth_cor = \ pyhessio_file.get_azimuth_cor(tel_id) data.mc.tel[tel_id].altitude_cor = \ pyhessio_file.get_altitude_cor(tel_id) yield data counter += 1 if max_events and counter >= max_events: pyhessio_file.close_file() return
def test_Provenance(): prov = Provenance() prov.start_activity("test1") prov.add_input_file("input.txt") prov.add_output_file("output.txt") prov.start_activity("test2") prov.add_input_file("input_a.txt") prov.add_input_file("input_b.txt") prov.finish_activity("test2") prov.finish_activity("test1") assert set(prov.finished_activity_names) == {'test2', 'test1'} return prov
def finish(self): self.hdulist.writeto(self.output_file, overwrite=self.overwrite) Provenance().add_output_file(self.output_file)
def finish(self): Provenance().add_output_file(self.output_file, role='mon.tel.calibration') self.writer.close()
def finish(self): Provenance().add_output_file(self.output_file, role='mon.tel.flatfield') self.writer.close()