def test_zle(self):
for w, pulse_bounds_should_be in (
([60, 60], [[0, 1]]),
([0, 60, 60, 0], [[0, 3]]),
([1] * 100 + [60] + [2] * 100, [[50, 149]]),
([1] * 100 + [30] + [2] * 100, []),
([1] * 100 + [60] + [2] * 200 + [60] + [3] * 100, [[50, 149],
[252, 351]]),
([1] * 100 + [60] + [2] * 70 + [60] + [3] * 100,
[[50, 100 + 1 + 70 + 1 + 50 - 1]]),
):
w = np.array(w).astype(np.int16)
# Convert from ADC above baseline (easier to specify) to raw ADC counts (what the plugin needs)
w = self.plugin.config['digitizer_reference_baseline'] - w
e = Event(
n_channels=self.plugin.config['n_channels'],
start_time=0,
stop_time=int(1e6),
sample_duration=self.pax.config['DEFAULT']['sample_duration'],
pulses=[Pulse(left=0, channel=1, raw_data=w)])
e = self.plugin.transform_event(e)
pulse_bounds = [[pulse.left, pulse.right] for pulse in e.pulses]
# Check the pulse bounds
self.assertEqual(pulse_bounds, pulse_bounds_should_be)
# Check if the data was put in correctly
for i, (l, r) in enumerate(pulse_bounds):
self.assertEqual(e.pulses[i].raw_data.tolist(),
w[l:r + 1].tolist())
def fake_events(n):
result = []
for i in range(n):
e = Event(n_channels=1, start_time=0, length=100, sample_duration=10)
e.event_number = i
e.block_id = i // 10
result.append(e)
return result
def transform_event(self, event_proxy):
t_start, t_end, run_start, collection_name = event_proxy.data
pulses, start = self.get_pulses(collection_name, t_start, t_end)
event_start = run_start + start
event = Event(n_channels = 8,
start_time = int(event_start),
length = max([p.right for p in pulses])+1,
sample_duration = self.config['sample_duration'],
event_number = event_proxy.event_number,
dataset_name = '',
block_id = event_proxy.block_id)
event.pulses = pulses
return event
def make_event(self):
"""Send the event from the currently processed pulses, then prepare for the next event
"""
event = Event(n_channels=self.config['n_channels'],
start_time=self.current_time + self.run_start_time,
sample_duration=self.dt,
stop_time=self.current_time +
self.dt * len(self.pulses[0].raw_data) +
self.run_start_time,
pulses=self.pulses,
event_number=self.events_built)
self.events_built += 1
self.pulses = []
return event
def transform_event(self, event_proxy):
# t0, t1 are the start, stop time of the event in pax units (ns) since the start of the run
(t0, t1), trigger_signals = event_proxy.data
self.log.debug("Fetching data for event with range [%s, %s]",
pax_to_human_time(t0),
pax_to_human_time(t1))
event = Event(n_channels=self.config['n_channels'],
block_id=event_proxy.block_id,
start_time=t0 + self.time_of_run_start,
sample_duration=self.sample_duration,
stop_time=t1 + self.time_of_run_start,
dataset_name=self.run_doc['name'],
event_number=event_proxy.event_number,
trigger_signals=trigger_signals)
# Convert trigger signal times to time since start of event
event.trigger_signals['left_time'] -= t0
event.trigger_signals['right_time'] -= t0
event.trigger_signals['time_mean'] -= t0
if self.split_collections:
start_col = self.subcollection_with_time(t0)
end_col = self.subcollection_with_time(t1)
if start_col == end_col:
count, mongo_iterator = self._get_cursor_between_times(t0, t1, start_col)
if count > self.max_pulses_per_event:
# Software "veto" the event to prevent overloading the event builder
if np.random.rand() > self.high_energy_prescale:
self.log.debug("VETO: %d pulses in event %s" % (len(event.pulses), event.event_number))
event.n_pulses = int(count)
return event
else:
self.log.info("Found event [%s-%s] which straddles subcollection boundary." % (
pax_to_human_time(t0), pax_to_human_time(t1)))
# Ignore the software-HEV in this case
mongo_iterator = chain(self._get_cursor_between_times(t0, t1, start_col)[1],
self._get_cursor_between_times(t0, t1, end_col)[1])
else:
mongo_iterator = self._get_cursor_between_times(t0, t1)
data_is_compressed = self.input_info['compressed']
for i, pulse_doc in enumerate(mongo_iterator):
digitizer_id = (pulse_doc['module'], pulse_doc['channel'])
pmt = self.pmt_mappings.get(digitizer_id)
if pmt is not None:
# Fetch the raw data
data = pulse_doc['data']
if data_is_compressed:
data = snappy.decompress(data)
time_within_event = self._from_mt(pulse_doc['time']) - t0 # ns
event.pulses.append(Pulse(left=self._to_mt(time_within_event),
raw_data=np.fromstring(data,
dtype="<i2"),
channel=pmt,
do_it_fast=True))
elif digitizer_id not in self.ignored_channels:
self.log.warning("Found data from digitizer module %d, channel %d,"
"which doesn't exist according to PMT mapping! Ignoring...",
pulse_doc['module'], pulse_doc['channel'])
self.ignored_channels.append(digitizer_id)
self.log.debug("%d pulses in event %s" % (len(event.pulses), event.event_number))
return event
def get_single_event_in_current_file(self, event_position=1):
# Seek to the requested event
# print(self.event_positions)
self.current_evefile.seek(self.event_positions[event_position])
# self.current_evefile.seek(event_position, whence=io.SEEK_CUR)
# Read event event header, check if it is a real data event or file event header or something different.
event_event_header = np.fromfile(self.current_evefile,
dtype=eve_event_header,
count=1)[0]
if event_event_header['event_type'] not in [
3, 4
]: # 3 = signal event, 4 = header event. Do others occur?
raise NotImplementedError(
"Event type %i not yet implemented!" %
event_event_header['event_type'], self.current_evefile.tell())
if event_event_header['event_type'] == 4:
# it might be possible to get another event header along with caen1724.par stuff
self.log.error(
"Unexpected event header at this position, trying to go on")
self.file_caen_pars = np.fromfile(self.current_evefile,
dtype=eve_caen1724_par_t,
count=1)[0]
event_event_header = np.fromfile(self.current_evefile,
dtype=eve_event_header,
count=1)[0]
# Start building the event
event = Event(
n_channels=14, # never trust the config file
start_time=int(event_event_header['event_timestamp'] * units.s # +
# event_layer_metadata['utc_time_usec'] * units.us
),
sample_duration=int(10 * units.ns),
# 10 ns is the inverse of the sampling frequency 10MHz
length=self.file_caen_pars['nof_samples'] # nof samples per event
)
event.dataset_name = self.current_filename # now metadata available
# as eve files do not have event numbers just count them
event.event_number = event_position
if self.file_caen_pars['zle'] == 0:
# Zero length encoding disabled
# Data is just a big bunch of samples from one channel, then next channel, etc
# unless board's last channel is read. Then signal header from next board and then again data
# Each channel has an equal number of samples.
for board_i, channels_active in enumerate(
self.file_caen_pars["chan_active"]):
if channels_active.sum(
) == 0: # if no channel is active there should be no signal header of the current board TODO: Check if that is really the case!
continue # skip the current board
event_signal_header_raw = np.fromfile(self.current_evefile,
dtype=eve_signal_header,
count=1)[0]
event_signal_header = header_unpacker(event_signal_header_raw)
for ch_i, channel_is_active in enumerate(channels_active):
if channel_is_active == 0:
continue # skip unused channels
chdata = np.fromfile(self.current_evefile,
dtype=np.int16,
count=int(
event_signal_header["page_size"]))
event.pulses.append(
Pulse(channel=ch_i + 8 * board_i,
left=0,
raw_data=np.array(chdata, dtype=np.int16)))
elif self.file_caen_pars['zle'] == 1:
# print(len(self.file_caen_pars["chan_active"]))
for board_i, channels_active in enumerate(
self.file_caen_pars["chan_active"]):
# Skip nonexistent board
if channels_active.sum(
) == 0: # if no channel is active there should be no signal header of the current board TODO: Check if that is really the case!
continue # skip the current board
event_signal_header_raw = np.fromfile(self.current_evefile,
dtype=eve_signal_header,
count=1)[0]
event_signal_header = header_unpacker(event_signal_header_raw)
channel_mask = event_signal_header["channel_mask"]
channels_included = [
i for i in range(8) if (2**i & channel_mask) > 0
]
for ch_i in channels_included: # enumerate(channels_active):
position = self.current_evefile.tell()
channel_size = np.fromfile(self.current_evefile,
dtype=np.uint32,
count=1)[0]
sample_position = 0
while (self.current_evefile.tell() <
position + channel_size * 4):
cword = np.fromfile(self.current_evefile,
dtype=np.uint32,
count=1)[0]
if cword < 0x80000000: # if cword is less than 0x80000000 waveform is below zle threshold
# skip word
sample_position += 2 * cword
continue
else:
chdata = np.fromfile(self.current_evefile,
dtype=np.int16,
count=2 *
(cword - 0x80000000))
event.pulses.append(
Pulse(channel=ch_i + 8 * board_i,
left=sample_position,
raw_data=chdata))
sample_position += 2 * (cword & (2**20 - 1))
# TODO: Check we have read all data for this event
affe = hex(
np.fromfile(self.current_evefile, dtype=np.uint32, count=1)[0])
if affe != '0xaffe':
print(
"WARNING : EVENT DID NOT END WITH 0XAFFE!! INSTEAD IT ENDED WITH ",
affe)
if event_position != len(self.event_positions) - 1:
current_pos = self.current_evefile.tell()
should_be_at_pos = self.event_positions[event_position + 1]
if current_pos != should_be_at_pos:
raise RuntimeError(
"Error during XED reading: after reading event %d from file "
"(event number %d) we should be at position %d, but we are at position %d!"
% (event_position, event.event_number, should_be_at_pos,
current_pos))
return event
def transform_event(self, event_proxy):
event_number = event_proxy.event_number
metadata = event_proxy.data['metadata']
xed_type = event_proxy.data['xed_type']
data = event_proxy.data['data']
event = Event(n_channels=self.config['n_channels'],
dataset_name=event_proxy.data['dataset_name'],
event_number=event_number,
block_id=event_proxy.block_id,
start_time=int(metadata['utc_time'] * units.s + metadata['utc_time_usec'] * units.us),
sample_duration=int(self.config['sample_duration']),
length=metadata['samples_in_event'])
if xed_type == 'raw':
data = np.reshape(data, (metadata['channels'], metadata['samples_in_event']))
for ch_i, chdata in enumerate(data):
event.pulses.append(Pulse(
channel=int(ch_i + 1), # +1 as first channel is 1 in Xenon100
left=0,
raw_data=chdata
))
elif xed_type == 'zle':
# Decompress event data into fake binary file (io.BytesIO)
try:
chunk_fake_file = six.BytesIO(bz2.decompress(data))
except (OSError, IOError):
# Maybe it wasn't compressed after all? We can at least try
# TODO: figure this out from flags
chunk_fake_file = six.BytesIO(data)
# Loop over all channels in the event to get the pulses
for channel_id in event_proxy.data['channels_included']:
# Read channel size (in 4bit words), subtract header size, convert
# from 4-byte words to bytes
channel_data_size = int(4 * (np.frombuffer(chunk_fake_file.read(4),
dtype='<u4')[0] - 1))
# Read the channel data into another fake binary file
channel_fake_file = six.BytesIO(chunk_fake_file.read(channel_data_size))
# Read the channel data control word by control word.
# sample_position keeps track of where in the waveform a new
# pulse should be placed.
sample_position = 0
while 1:
# Is there a new control word?
control_word_string = channel_fake_file.read(4)
if not control_word_string:
break
# Control words starting with zero indicate a number of sample PAIRS to skip
control_word = int(np.frombuffer(control_word_string,
dtype='<u4')[0])
if control_word < 2 ** 31:
sample_position += 2 * control_word
continue
# Control words starting with one indicate a number of sample PAIRS follow
else:
# Subtract the control word flag
data_samples = 2 * (control_word - (2 ** 31))
# Note endianness
samples_pulse = np.frombuffer(channel_fake_file.read(2 * data_samples),
dtype="<i2")
event.pulses.append(Pulse(
channel=int(channel_id),
left=int(sample_position),
raw_data=samples_pulse
))
sample_position += len(samples_pulse)
return event
