def load_r1(self, fits_file, fits_mode=1, test=False):
if not fits_file:
self.log.error("No FITS file provided")
return False
if not os.path.isfile(fits_file):
self.log.error("Bad file provided")
return False
self.log.info("Loading FITS file %s in mode %d", fits_file, fits_mode)
if fits_mode == 0:
try:
f = fits.open(fits_file)
except:
self.log.error("Invalid FITS file provided")
return False
sc = self.__load_schema_from_astropy(f, test)
f.close()
return sc
if fits_mode == 1:
try:
f = File(fits_file)
except OSError:
self.log.error("Invalid FITS file provided")
return False
sc = self.__load_schema_from_protozfits(f, test)
f.close()
return sc
def __init__(self, input_url):
self._file = {}
self._events = {}
self._events_table = {}
self._camera_config = {}
paths = [
input_url,
]
# test how many files are there
if '000.fits.fz' in input_url:
i = 0
while True:
input_url = input_url.replace(
str(i).zfill(3) + '.fits.fz',
str(i + 1).zfill(3) + '.fits.fz')
if exists(input_url):
paths.append(input_url)
# keep track of all input files
Provenance().add_input_file(input_url, role='dl0.sub.evt')
i = i + 1
else:
break
# open the files and get the first fits Tables
from protozfits import File
for path in paths:
self._file[path] = File(path)
self._events_table[path] = File(path).Events
try:
self._events[path] = next(self._file[path].Events)
self._camera_config[path] = next(self._file[path].CameraConfig)
except StopIteration:
pass
# verify that the configuration_id of all files are the same
# in the CameraConfig table
for path in paths:
assert (self._camera_config[path].configuration_id ==
self._camera_config[paths[0]].configuration_id)
# keep the cameraConfig of the first file
self.camera_config = self._camera_config[paths[0]]
def __init__(self, config=None, tool=None, **kwargs):
super().__init__(config=config, tool=tool, **kwargs)
from protozfits import File
self.file = File(self.input_url)
# TODO: Correct pixel ordering
self._tel_desc = TelescopeDescription.from_name(optics_name='SST-1M',
camera_name='DigiCam')
def test_File_getitem_with_range():
f = File(example_file_path)
interesting_event_ids = range(9, 1, -2)
expected_event_numbers = [
FIRST_EVENT_NUMBER + i for i in interesting_event_ids
]
for i, event in enumerate(f.Events[interesting_event_ids]):
assert event.eventNumber == expected_event_numbers[i]
def test_File_getitem_with_slice():
f = File(example_file_path)
expected_event_numbers = [
FIRST_EVENT_NUMBER + 1,
FIRST_EVENT_NUMBER + 2,
]
for i, event in enumerate(f.Events[1:3]):
assert event.eventNumber == expected_event_numbers[i]
def test_File_getitem_with_iterable():
f = File(example_file_path)
expected_event_numbers = [
FIRST_EVENT_NUMBER + 3,
FIRST_EVENT_NUMBER + 7,
FIRST_EVENT_NUMBER + 1,
]
for i, event in enumerate(f.Events[[3, 7, 1]]):
assert event.eventNumber == expected_event_numbers[i]
def test_can_iterate_over_events_and_run_header():
with File(example_file_path) as f:
camera_config = next(f.CameraConfig)
assert (camera_config.expected_pixels_id == np.arange(14)).all()
for i, event in enumerate(f.Events):
assert event.event_id == i + 1
assert event.waveform.shape == (1120, )
assert event.pixel_status.shape == (14, )
assert event.lstcam.first_capacitor_id.shape == (16, )
assert event.lstcam.counters.shape == (44, )
def __init__(self, file_list):
self._file = {}
self._events = {}
self._events_table = {}
self._camera_config = {}
self.camera_config = None
paths = []
for file_name in file_list:
paths.append(file_name)
Provenance().add_input_file(file_name, role='r0.sub.evt')
# open the files and get the first fits Tables
from protozfits import File
for path in paths:
try:
self._file[path] = File(path)
self._events_table[path] = File(path).Events
self._events[path] = next(self._file[path].Events)
# verify where the CameraConfig is present
if 'CameraConfig' in self._file[path].__dict__.keys():
self._camera_config[path] = next(
self._file[path].CameraConfig)
# for the moment it takes the first CameraConfig it finds (to be changed)
if (self.camera_config is None):
self.camera_config = self._camera_config[path]
except StopIteration:
pass
# verify that somewhere the CameraConfing is present
assert self.camera_config
def __init__(self, paths):
"""
Create a new MultiFiles object from an iterable of paths
Parameters
----------
paths: Iterable[string|Path]
The input paths
"""
paths = list(paths)
if len(paths) == 0:
raise ValueError('`paths` must not be empty')
self._file = {}
self._events = {}
self._events_table = {}
self._camera_config = {}
for path in paths:
Provenance().add_input_file(path, role='r0.sub.evt')
try:
self._file[path] = File(str(path))
self._events_table[path] = self._file[path].Events
self._events[path] = next(self._file[path].Events)
if hasattr(self._file[path], 'CameraConfig'):
self._camera_config[path] = next(self._file[path].CameraConfig)
else:
warnings.warn(f'No CameraConfig found in {path}')
except StopIteration:
pass
run_ids = {
config.configuration_id
for config in self._camera_config.values()
}
if len(run_ids) > 1:
raise IOError(f'Found multiple run_ids: {run_ids}')
# verify that we found a CameraConfig
if len(self._camera_config) == 0:
raise IOError(f"No CameraConfig was found in any of the input files: {paths}")
else:
self.camera_config = next(iter(self._camera_config.values()))
def __init__(self, zfits_filename, n_channel=2, n_sample=4320, n_synchro_max=None):
self._file = ZfitsFile(zfits_filename)
self.generator = self._file.Events
self.events = {}
self.synchronized = []
self.not_synchronized = []
self.n_channel = n_channel
self.n_sample = n_sample
event = next(self.generator)
while True:
self.add_camera_event(event)
if n_synchro_max is not None:
if len(self.synchronized) >= n_synchro_max:
break
try:
event = next(self.generator)
except StopIteration:
break
def test_can_open_and_get_an_event_from_all_test_resources():
print()
for path in all_test_resources:
with File(path) as f:
event = next(f.Events)
print(path, len(str(event)))
class ZfitsDatafile:
def __init__(self,
zfits_filename,
n_channel=2,
n_sample=4320,
n_synchro_max=None):
self._file = ZfitsFile(zfits_filename)
self.generator = self._file.Events
self.events = {}
self.synchronized = []
self.not_synchronized = []
self.n_channel = n_channel
self.n_sample = n_sample
event = next(self.generator)
while True:
self.add_camera_event(event)
if n_synchro_max is not None:
if len(self.synchronized) >= n_synchro_max:
break
try:
event = next(self.generator)
except StopIteration:
break
def __del__(self):
self._file.close()
self.events = {}
self.synchronized = []
self.not_synchronized = []
def add_camera_event(self, event, disable_different_timing=True):
evt_nb = event.eventNumber
samples = event.sii.samples
assert self.n_sample == np.size(samples)
if evt_nb in self.events.keys():
acq = self.events[evt_nb]
if event.local_time_nanosec != acq.time_ns or event.local_time_sec != acq.time_s:
if disable_different_timing:
return
print("WARNING: error of",
event.local_time_nanosec - acq.time_ns,
"ns synchronizing event", evt_nb)
acq.set_channel(event.sii.channelId, samples)
nrecorded_channel = len(acq.filled_channels)
if nrecorded_channel == self.n_channel:
self.not_synchronized.remove(evt_nb)
self.synchronized.append(evt_nb)
elif nrecorded_channel == 1:
self.not_synchronized.append(evt_nb)
else:
self.events[evt_nb] = SiiAcquisition(
time_sec=event.local_time_sec,
time_nanosec=event.local_time_nanosec,
channel=event.sii.channelId,
wf=samples,
n_channel=self.n_channel)
self.not_synchronized.append(evt_nb)
def remove(self, evt_nb):
self.events.pop(evt_nb)
if evt_nb in self.synchronized:
self.synchronized.remove(evt_nb)
else:
self.not_synchronized.remove(evt_nb)
def add_sii_acquisition(self,
evt_nb,
event,
disable_different_timing=True):
assert isinstance(event, SiiAcquisition)
if evt_nb in self.events.keys():
existing = self.events[evt_nb]
if event.time_ns != existing.time_ns or event.time_s != existing.time_s:
if disable_different_timing:
return
print("WARNING: error of", event.time_ns - existing.time_ns,
"ns synchronizing event", evt_nb)
for channel in event.filled_channels:
existing.set_channel(channel, event.wf[channel])
if existing.is_synchronized():
self.not_synchronized.remove(evt_nb)
self.synchronized.append(evt_nb)
else:
first_filled_channel = event.filled_channels[0]
self.events[evt_nb] = SiiAcquisition(
time_sec=event.time_s,
time_nanosec=event.time_ns,
channel=first_filled_channel,
wf=event.wf[first_filled_channel, :],
n_channel=self.n_channel)
for channel in event.filled_channels[1:]:
self.events[evt_nb].set_channel(channel, event.wf[channel, :])
if self.events[evt_nb].is_synchronized():
self.synchronized.append(evt_nb)
else:
self.not_synchronized.append(evt_nb)
def add_datafile(self, datafile):
self.generator = datafile.generator
for evt_nb in datafile.events.keys():
event = datafile.events[evt_nb]
self.add_sii_acquisition(evt_nb, event)
def get_batch_synchronized(self, batch_size, remove_events=False):
synchronized = self.synchronized
if len(synchronized) < batch_size:
event = next(self.generator)
while len(self.synchronized) < batch_size:
self.add_camera_event(event)
try:
event = next(self.generator)
except StopIteration:
break
if len(synchronized) < batch_size:
print(
"WARNING: not enough synchronized events to make a full batch")
batch_waveform = np.zeros(
[len(synchronized), self.n_sample, self.n_channel])
else:
batch_waveform = np.zeros(
[batch_size, self.n_sample, self.n_channel])
event_read = []
for batch_idx, evt_nb in enumerate(synchronized):
if batch_idx == batch_size:
break
batch_waveform[batch_idx, :, :] = self.events[evt_nb].wf.T
event_read.append(evt_nb)
if remove_events:
for evt_nb in event_read:
self.remove(evt_nb)
return batch_waveform
def test_File_getitem_with_integer():
f = File(example_file_path)
event = f.Events[0]
assert event.eventNumber == FIRST_EVENT_NUMBER
def count_number_events(file_list):
return sum(len(File(filename).Events) for filename in file_list)
def __init__(self, config=None, tool=None, **kwargs):
super().__init__(config=config, tool=tool, **kwargs)
from protozfits import File
self.file = File(self.input_url)
self.header = next(self.file.RunHeader)
def zfits_event_source(url,
camera=camera.DigiCam,
max_events=None,
allowed_tels=None,
event_id=None,
disable_bar=False):
"""A generator that streams data from an ZFITs data file
Parameters
----------
url : str
path to file to open
max_events : int, optional
maximum number of events to read
allowed_tels : list[int]
select only a subset of telescope, if None, all are read. This can
be used for example emulate the final CTA data format, where there
would be 1 telescope per file (whereas in current monte-carlo,
they are all interleaved into one file)
camera : digicampipe.instrument.Camera(), default DigiCam
event_id: int
Event id to start at. If the exact event ID does not exists
it will return the closest past event. If the event ID is out of the
range of the file it will raise an IndexError
disable_bar: If set to true, the progress bar is not shown.
"""
data = DataContainer()
with File(url) as file:
loaded_telescopes = []
n_events_in_file = len(file.Events)
events = file.Events
index_of_event = 0
if event_id is not None:
index_of_event = _binary_search(file, event_id)
first_event_id = file.Events[0].eventNumber
last_event_id = file.Events[n_events_in_file - 1].eventNumber
if not first_event_id <= event_id <= last_event_id:
raise IndexError('Cannot find event ID {} in File {}\n'
'First event ID : {}\n'
'Last event ID : {}'.format(
event_id, url, first_event_id,
last_event_id))
events = events[max(index_of_event, 0):]
n_steps = n_events_in_file if max_events is None else max_events
for event_counter, event in tqdm(enumerate(events),
desc='Events',
leave=True,
initial=index_of_event,
disable=disable_bar,
total=n_steps):
if max_events is not None and event_counter > max_events:
break
data.r0.event_id = event_counter
data.r0.tels_with_data = [
event.telescopeID,
]
# remove forbidden telescopes
if allowed_tels:
data.r0.tels_with_data = [
list(filter(lambda x: x in data.r0.tels_with_data,
sublist)) for sublist in allowed_tels
]
_sort_ids = None
for tel_id in data.r0.tels_with_data:
pixel_ids = event.hiGain.waveforms.pixelsIndices
n_pixels = len(pixel_ids)
if _sort_ids is None:
_sort_ids = np.argsort(pixel_ids)
samples = event.hiGain.waveforms.samples.reshape(n_pixels, -1)
try:
unsorted_baseline = event.hiGain.waveforms.baselines
except AttributeError:
warnings.warn(
("Could not read `hiGain.waveforms.baselines`"
"for event:{}\n"
"of file:{}\n".format(event_counter, url)))
return np.ones(n_pixels) * np.nan
if tel_id not in loaded_telescopes:
data.inst.num_channels[tel_id] = event.num_gains
data.inst.geom[tel_id] = camera.geometry
data.inst.cluster_matrix_7[tel_id] = \
camera.cluster_7_matrix
data.inst.cluster_matrix_19[tel_id] = \
camera.cluster_19_matrix
data.inst.patch_matrix[tel_id] = camera.patch_matrix
data.inst.num_pixels[tel_id] = samples.shape[0]
data.inst.num_samples[tel_id] = samples.shape[1]
loaded_telescopes.append(tel_id)
r0 = data.r0.tel[tel_id]
r0.camera_event_number = event.eventNumber
r0.pixel_flags = event.pixels_flags[_sort_ids]
r0.local_camera_clock = (np.int64(event.local_time_sec * 1E9) +
np.int64(event.local_time_nanosec))
r0.gps_time = (np.int64(event.trig.timeSec * 1E9) +
np.int64(event.trig.timeNanoSec))
r0.camera_event_type = event.event_type
r0.array_event_type = event.eventType
r0.adc_samples = samples[_sort_ids]
if len(event.trigger_input_traces) > 0:
r0.trigger_input_traces = _prepare_trigger_input(
event.trigger_input_traces)
else:
warnings.warn(
'trigger_input_traces does not exist: --> nan')
r0.trigger_input_traces = np.zeros(
(432, data.inst.num_samples[tel_id])) * np.nan
if len(event.trigger_output_patch7) > 0:
r0.trigger_output_patch7 = _prepare_trigger_output(
event.trigger_output_patch7)
else:
warnings.warn(
'trigger_output_patch7 does not exist: --> nan')
r0.trigger_output_patch7 = np.zeros(
(432, data.inst.num_samples[tel_id])) * np.nan
if len(event.trigger_output_patch19) > 0:
r0.trigger_output_patch19 = _prepare_trigger_output(
event.trigger_output_patch19)
else:
warnings.warn(
'trigger_output_patch19 does not exist: --> nan')
r0.trigger_output_patch19 = np.zeros(
(432, data.inst.num_samples[tel_id])) * np.nan
r0.digicam_baseline = unsorted_baseline[_sort_ids] / 16
yield data
def func():
for _, i in zip(File(example_file_path).Events, range(100)):
pass
assert i == 99