def test_provenence_contextmanager():
prov = Provenance()
with prov.activity("myactivity"):
assert 'myactivity' in prov.active_activity_names
assert 'myactivity' in prov.finished_activity_names
assert 'myactivity' not in prov.active_activity_names
def save(self):
"""
Save the figure to the path defined by the `output_path` trait
"""
self._finish()
output_dir = os.path.dirname(self.output_path)
if not os.path.exists(output_dir):
print(f"Creating directory: {output_dir}")
os.makedirs(output_dir)
self.fig.savefig(self.output_path, bbox_inches='tight')
print(f"Figure saved to: {self.output_path}")
Provenance().add_output_file(self.output_path)
plt.close(self.fig)
def finish(self):
"""
finish up and write out results (called automatically after
`start()`)
"""
# write out the final table
if self.outfile.endswith('fits') or self.outfile.endswith('fits.gz'):
self.events.write(self.outfile, overwrite=self.overwrite)
elif self.outfile.endswith('h5'):
self.events.write(self.outfile, path='/events',
overwrite=self.overwrite)
else:
self.events.write(self.outfile)
Provenance().add_output_file(self.outfile)
self.log.info("Table written to '{}'".format(self.outfile))
self.log.info('\n %s', self.events)
def setup(self):
self.filename_dl3 = dl2_to_dl3_filename(self.input_dl2,
compress=self.gzip)
self.provenance_log = self.output_dl3_path / (self.name +
".provenance.log")
Provenance().add_input_file(self.input_dl2)
self.event_sel = EventSelector(parent=self)
self.cuts = DL3Cuts(parent=self)
self.output_file = self.output_dl3_path.absolute() / self.filename_dl3
if self.output_file.exists():
if self.overwrite:
self.log.warning(f"Overwriting {self.output_file}")
self.output_file.unlink()
else:
raise ToolConfigurationError(
f"Output file {self.output_file} already exists,"
" use --overwrite to overwrite")
if not (self.source_ra or self.source_dec):
self.source_pos = SkyCoord.from_name(self.source_name)
elif bool(self.source_ra) != bool(self.source_dec):
raise ToolConfigurationError(
"Either provide both RA and DEC values for the source or none")
else:
self.source_pos = SkyCoord(ra=self.source_ra, dec=self.source_dec)
self.log.debug(f"Output DL3 file: {self.output_file}")
try:
with fits.open(self.input_irf) as hdul:
self.use_energy_dependent_gh_cuts = (
"GH_CUT" not in hdul["EFFECTIVE AREA"].header)
except:
raise ToolConfigurationError(
f"{self.input_irf} does not have EFFECTIVE AREA HDU, "
" to check for global cut information in the Header value")
if self.source_dep:
with fits.open(self.input_irf) as hdul:
self.use_energy_dependent_alpha_cuts = (
"AL_CUT" not in hdul["EFFECTIVE AREA"].header)
def setup(self):
self.list_files = sorted(self.input_dl3_dir.glob(self.file_pattern))
if len(self.list_files) == 0:
raise ToolConfigurationError(
f"No files found with pattern {self.file_pattern} in {self.input_dl3_dir}"
)
for f in self.list_files:
Provenance().add_input_file(f)
if not self.output_index_path:
self.output_index_path = self.input_dl3_dir
self.hdu_index_file = self.output_index_path / self.hdu_index_filename
self.obs_index_file = self.output_index_path / self.obs_index_filename
self.provenance_log = self.output_index_path / (self.name + ".provenance.log")
if self.hdu_index_file.exists():
if self.overwrite:
self.log.warning(f"Overwriting {self.hdu_index_file}")
self.hdu_index_file.unlink()
else:
raise ToolConfigurationError(
f"Output file {self.hdu_index_file} already exists,"
"use --overwrite to overwrite"
)
if self.obs_index_file.exists():
if self.overwrite:
self.log.warning(f"Overwriting {self.obs_index_file}")
self.obs_index_file.unlink()
else:
raise ToolConfigurationError(
f"Output file {self.obs_index_file} already exists,"
" use --overwrite to overwrite"
)
self.log.debug("HDU Index file: %s", self.hdu_index_file)
self.log.debug("OBS Index file: %s", self.obs_index_file)
def write_camera_geometries(self):
cam_types = get_camera_types(self.inst.subarray)
self.inst.subarray.info(printer=self.log.info)
for cam_name in cam_types:
ext, args = self._get_file_format_info(self.format,
'CAMGEOM',
cam_name)
self.log.debug("writing {}".format(cam_name))
tel_id = cam_types[cam_name].pop()
geom = self.inst.subarray.tel[tel_id].camera
table = geom.to_table()
table.meta['SOURCE'] = self.infile
filename = "{}.camgeom.{}".format(cam_name, ext)
try:
table.write(filename, **args)
Provenance().add_output_file(filename, 'dl0.tel.svc.camera')
except IOError as err:
self.log.warn("couldn't write camera definition '%s' because: "
"%s", filename, err)
def write_camera_geometries(self):
cam_types = get_camera_types(self.subarray)
self.subarray.info(printer=self.log.info)
for cam_name in cam_types:
ext, args = self._get_file_format_info(self.format, "CAMGEOM",
cam_name)
self.log.debug(f"writing {cam_name}")
tel_id = cam_types[cam_name].pop()
geom = self.subarray.tel[tel_id].camera.geometry
table = geom.to_table()
table.meta["SOURCE"] = str(self.infile)
filename = f"{cam_name}.camgeom.{ext}"
try:
table.write(filename, **args)
Provenance().add_output_file(filename, "dl0.tel.svc.camera")
except IOError as err:
self.log.warning(
"couldn't write camera definition '%s' because: %s",
filename, err)
def finish(self):
"""
finish up and write out results (called automatically after
`start()`)
"""
# write out the final table
try:
if '.fits' in self.outfile.suffixes:
self.events.write(self.outfile, overwrite=self.overwrite)
elif self.outfile.suffix in ('.hdf5', '.h5', '.hdf'):
self.events.write(self.outfile,
path='/events',
overwrite=self.overwrite)
else:
self.events.write(self.outfile)
Provenance().add_output_file(self.outfile)
except IOError as err:
self.log.warning("Couldn't write output (%s)", err)
self.log.info('\n %s', self.events)
def finish(self):
"""
finish up and write out results (called automatically after
`start()`)
"""
# write out the final table
try:
if self.outfile.endswith('fits') or self.outfile.endswith(
'fits.gz'):
self.events.write(self.outfile, overwrite=self.overwrite)
elif self.outfile.endswith('h5'):
self.events.write(self.outfile,
path='/events',
overwrite=self.overwrite)
else:
self.events.write(self.outfile)
Provenance().add_output_file(self.outfile)
except IOError as err:
self.log.warn("Couldn't write output (%s)", err)
self.log.info('\n %s', self.events)
def __init__(self, file_list):
self._file = {}
self._events = {}
self._events_table = {}
self._camera_config = {}
self.camera_config = None
paths = []
for file_name in file_list:
paths.append(file_name)
Provenance().add_input_file(file_name, role='r0.sub.evt')
# open the files and get the first fits Tables
from protozfits import File
for path in paths:
try:
self._file[path] = File(path)
self._events_table[path] = File(path).Events
self._events[path] = next(self._file[path].Events)
# verify where the CameraConfig is present
if 'CameraConfig' in self._file[path].__dict__.keys():
self._camera_config[path] = next(
self._file[path].CameraConfig)
# for the moment it takes the first CameraConfig it finds (to be changed)
if (self.camera_config is None):
self.camera_config = self._camera_config[path]
except StopIteration:
pass
# verify that somewhere the CameraConfing is present
assert self.camera_config
def hessio_get_list_event_ids(url, max_events=None):
"""
Faster method to get a list of all the event ids in the hessio file.
This list can also be used to find out the number of events that exist
in the file.
Parameters
----------
url : str
path to file to open
max_events : int, optional
maximum number of events to read
Returns
-------
event_id_list : list[num_events]
A list with all the event ids that are in the file.
"""
logger.warning("This method is slow. Need to find faster method.")
try:
with open_hessio(url) as pyhessio_file:
Provenance().add_input_file(url, role='r0.sub.evt')
counter = 0
event_id_list = []
eventstream = pyhessio_file.move_to_next_event()
for event_id in eventstream:
event_id_list.append(event_id)
counter += 1
if max_events and counter >= max_events:
pyhessio_file.close_file()
break
return event_id_list
except HessioError:
raise RuntimeError(
"hessio_event_source failed to open '{}'".format(url))
def finish(self):
Provenance().add_output_file(self.outfile,
role='dl1.tel.evt.muon')
self.writer.close()
def finish(self):
Provenance().add_output_file(self.hdu_index_file)
Provenance().add_output_file(self.obs_index_file)
def finish(self):
fits.HDUList(self.hdus).writeto(self.output_irf_file,
overwrite=self.overwrite)
Provenance().add_output_file(self.output_irf_file)
def test_Provenance():
prov = Provenance()
prov.start_activity("test1")
prov.add_input_file("input.txt")
prov.add_output_file("output.txt")
prov.start_activity("test2")
prov.add_input_file("input_a.txt")
prov.add_input_file("input_b.txt")
prov.finish_activity("test2")
prov.finish_activity("test1")
assert set(prov.finished_activity_names) == {"test2", "test1"}
return prov
def load_camera_geometry(version=4):
''' Load camera geometry from bundled resources of this repo '''
f = resource_filename('ctapipe_io_lst',
f'resources/LSTCam-{version:03d}.camgeom.fits.gz')
Provenance().add_input_file(f, role="CameraGeometry")
return CameraGeometry.from_table(f)
def finish(self):
Provenance().add_output_file(
self.output_file,
role='mon.tel.pedestal'
)
self.writer.close()
def finish(self):
Provenance().add_output_file(self.output,
role="muon_efficiency_parameters")
self.writer.close()
def __init__(self, **kwargs):
"""
Constructor
Parameters
----------
n_gains = number of gains expected in input file
baseline = baseline to be subtracted at r1 level (not used for the moment)
multi_streams = enable the reading of input files from all streams
config: traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.\
kwargs: dict
Additional parameters to be passed.
NOTE: The file mask of the data to read can be passed with
the 'input_url' parameter.
"""
# EventSource can not handle file wild cards as input_url
# To overcome this we substitute the input_url with first file matching
# the specified file mask (copied from MAGICEventSourceROOT).
super().__init__(**kwargs)
if self.multi_streams:
# test how many streams are there:
# file name must be [stream name]Run[all the rest]
# All the files with the same [all the rest] are opened
if '/' in self.input_url:
dir, name = self.input_url.rsplit('/', 1)
else:
dir = getcwd()
name = self.input_url
if 'Run' in name:
stream, run = name.split('Run', 1)
else:
run = name
ls = listdir(dir)
self.file_list = []
for file_name in ls:
if run in file_name:
full_name = dir + '/' + file_name
self.file_list.append(full_name)
Provenance().add_input_file(full_name, role='dl0.sub.evt')
else:
self.file_list = [self.input_url]
self.multi_file = MultiFiles(self.file_list)
self.camera_config = self.multi_file.camera_config
self.log.info(
"Read {} input files".format(
self.multi_file.num_inputs()
)
)
def hessio_event_source(url,
camera_geometry,
camera,
max_events=None,
allowed_tels=None,
requested_event=None,
use_event_id=False):
"""A generator that streams data from an EventIO/HESSIO MC data file
(e.g. a standard CTA data file.)
Parameters
----------
url : str
path to file to open
max_events : int, optional
maximum number of events to read
allowed_tels : list[int]
select only a subset of telescope, if None, all are read. This can
be used for example emulate the final CTA data format, where there
would be 1 telescope per file (whereas in current monte-carlo,
they are all interleaved into one file)
requested_event : int
Seek to a paricular event index
use_event_id : bool
If True ,'requested_event' now seeks for a particular event id instead
of index
"""
with open_hessio(url) as pyhessio_file:
# the container is initialized once, and data is replaced within
# it after each yield
Provenance().add_input_file(url, role='dl0.sub.evt')
counter = 0
eventstream = pyhessio_file.move_to_next_event()
if allowed_tels is not None:
allowed_tels = set(allowed_tels)
data = DataContainer()
data.meta['origin'] = "hessio"
# some hessio_event_source specific parameters
data.meta['input'] = url
data.meta['max_events'] = max_events
for event_id in eventstream:
# Seek to requested event
if requested_event is not None:
current = counter
if use_event_id:
current = event_id
if not current == requested_event:
counter += 1
continue
data.r0.run_id = pyhessio_file.get_run_number()
data.r0.event_id = event_id
data.r0.tels_with_data = list(pyhessio_file.get_teldata_list())
data.r1.run_id = pyhessio_file.get_run_number()
data.r1.event_id = event_id
data.r1.tels_with_data = list(pyhessio_file.get_teldata_list())
data.dl0.run_id = pyhessio_file.get_run_number()
data.dl0.event_id = event_id
data.dl0.tels_with_data = list(pyhessio_file.get_teldata_list())
# handle telescope filtering by taking the intersection of
# tels_with_data and allowed_tels
if allowed_tels is not None:
selected = data.r0.tels_with_data & allowed_tels
if len(selected) == 0:
continue # skip event
data.r0.tels_with_data = selected
data.r1.tels_with_data = selected
data.dl0.tels_with_data = selected
data.trig.tels_with_trigger \
= pyhessio_file.get_central_event_teltrg_list()
time_s, time_ns = pyhessio_file.get_central_event_gps_time()
data.trig.gps_time = Time(time_s * u.s,
time_ns * u.ns,
format='unix',
scale='utc')
data.mc.energy = pyhessio_file.get_mc_shower_energy() * u.TeV
data.mc.alt = Angle(pyhessio_file.get_mc_shower_altitude(), u.rad)
data.mc.az = Angle(pyhessio_file.get_mc_shower_azimuth(), u.rad)
data.mc.core_x = pyhessio_file.get_mc_event_xcore() * u.m
data.mc.core_y = pyhessio_file.get_mc_event_ycore() * u.m
first_int = pyhessio_file.get_mc_shower_h_first_int() * u.m
data.mc.h_first_int = first_int
# mc run header data
data.mcheader.run_array_direction = \
pyhessio_file.get_mc_run_array_direction()
data.count = counter
# this should be done in a nicer way to not re-allocate the
# data each time (right now it's just deleted and garbage
# collected)
data.r0.tel.clear()
data.r1.tel.clear()
data.dl0.tel.clear()
data.dl1.tel.clear()
data.mc.tel.clear() # clear the previous telescopes
_fill_instrument_info(data, pyhessio_file, camera_geometry, camera)
for tel_id in data.r0.tels_with_data:
data.mc.mc_event_offset_fov = \
pyhessio_file.get_mc_event_offset_fov()
data.mc.tel[tel_id].dc_to_pe \
= pyhessio_file.get_calibration(tel_id)
data.mc.tel[tel_id].pedestal \
= pyhessio_file.get_pedestal(tel_id)
data.r0.tel[tel_id].adc_samples = \
pyhessio_file.get_adc_sample(tel_id)
data.r0.tel[tel_id].adc_samples = \
data.r0.tel[tel_id].adc_samples[0]
data.r0.tel[tel_id].local_camera_clock = 0
if data.r0.tel[tel_id].adc_samples.size == 0:
# To handle ASTRI and dst files
data.r0.tel[tel_id].adc_samples = \
pyhessio_file.get_adc_sum(tel_id)[..., None]
data.r0.tel[tel_id].adc_sums = \
pyhessio_file.get_adc_sum(tel_id)
data.mc.tel[tel_id].reference_pulse_shape = \
pyhessio_file.get_ref_shapes(tel_id)
nsamples = pyhessio_file.get_event_num_samples(tel_id)
if nsamples <= 0:
nsamples = 1
data.r0.tel[tel_id].num_samples = nsamples
# load the data per telescope/pixel
hessio_mc_npe = pyhessio_file.get_mc_number_photon_electron
data.mc.tel[tel_id].photo_electron_image \
= hessio_mc_npe(telescope_id=tel_id)
data.mc.tel[tel_id].meta['refstep'] = \
pyhessio_file.get_ref_step(tel_id)
data.mc.tel[tel_id].time_slice = \
pyhessio_file.get_time_slice(tel_id)
data.mc.tel[tel_id].azimuth_raw = \
pyhessio_file.get_azimuth_raw(tel_id)
data.mc.tel[tel_id].altitude_raw = \
pyhessio_file.get_altitude_raw(tel_id)
data.mc.tel[tel_id].azimuth_cor = \
pyhessio_file.get_azimuth_cor(tel_id)
data.mc.tel[tel_id].altitude_cor = \
pyhessio_file.get_altitude_cor(tel_id)
yield data
counter += 1
if max_events and counter >= max_events:
pyhessio_file.close_file()
return
def test_Provenance():
prov = Provenance()
prov.start_activity("test1")
prov.add_input_file("input.txt")
prov.add_output_file("output.txt")
prov.start_activity("test2")
prov.add_input_file("input_a.txt")
prov.add_input_file("input_b.txt")
prov.finish_activity("test2")
prov.finish_activity("test1")
assert set(prov.finished_activity_names) == {'test2', 'test1'}
return prov
def finish(self):
self.hdulist.writeto(self.output_file, overwrite=self.overwrite)
Provenance().add_output_file(self.output_file)
def finish(self):
Provenance().add_output_file(self.output_file,
role='mon.tel.calibration')
self.writer.close()
def finish(self):
Provenance().add_output_file(self.output_file,
role='mon.tel.flatfield')
self.writer.close()
