def test_from_break(records):
window = 5
def has_break(x):
if len(x) < 2:
return False
return np.diff(x['time']).max() > window
try:
left = strax.from_break(records,
safe_break=window,
left=True,
tolerant=False)
right = strax.from_break(records,
safe_break=window,
left=False,
tolerant=False)
except strax.NoBreakFound:
assert not has_break(records)
else:
assert len(left) + len(right) == len(records)
if len(records) > 0:
np.testing.assert_equal(np.concatenate([left, right]), records)
if len(left) and len(right):
assert left[-1]['time'] <= right[0]['time'] - window
assert not has_break(right)
def test_from_break(records):
if not len(records):
return
window = 5
def has_break(x):
if len(x) < 2:
return False
for i in range(1, len(x)):
if strax.endtime(x[:i]).max() + window <= x[i]['time']:
return True
return False
try:
left, t_break_l = strax.from_break(records,
safe_break=window,
left=True,
tolerant=False)
right, t_break_r = strax.from_break(records,
safe_break=window,
left=False,
tolerant=False)
except strax.NoBreakFound:
assert not has_break(records)
else:
assert t_break_l == t_break_r, "Inconsistent break time"
t_break = t_break_l
assert has_break(records), f"Found nonexistent break at {t_break}"
assert len(left) + len(right) == len(records), "Data loss"
if len(records) > 0:
np.testing.assert_equal(np.concatenate([left, right]), records)
if len(right) and len(left):
assert t_break == right[0]['time']
assert strax.endtime(left).max() <= right[0]['time'] - window
assert not has_break(right)
def _load_chunk(self, path, kind='central'):
records = [
strax.load_file(fn, compressor='blosc', dtype=strax.record_dtype())
for fn in glob.glob(f'{path}/reader_*')
]
records = np.concatenate(records)
records = strax.sort_by_time(records)
if kind == 'central':
return records
result = strax.from_break(
records,
safe_break=int(1e3), # TODO config?
left=kind == 'post',
tolerant=True)
if self.config['erase']:
shutil.rmtree(path)
return result
def _load_chunk(self, path, kind='central'):
records = [
strax.load_file(fn, compressor='blosc', dtype=strax.record_dtype())
for fn in sorted(glob.glob(f'{path}/*'))
]
records = np.concatenate(records)
records = strax.sort_by_time(records)
if kind == 'central':
result = records
else:
result = strax.from_break(
records,
safe_break=self.config['safe_break_in_pulses'],
left=kind == 'post',
tolerant=True)
if self.config['erase']:
shutil.rmtree(path)
return result
def load_chunk(self, folder, kind='central'):
records = np.concatenate([
strax.load_file(os.path.join(folder, f),
compressor='blosc',
dtype=strax.record_dtype())
for f in os.listdir(folder)
])
records = strax.sort_by_time(records)
if kind == 'central':
result = records
else:
if self.config['do_breaks']:
result = strax.from_break(records,
safe_break=self.config['safe_break'],
left=kind == 'post',
tolerant=True)
else:
result = records
result['time'] += self.config['run_start']
return result
def _load_chunk(self, path, start, end, kind='central'):
records = [
strax.load_file(fn,
compressor=self.config["daq_compressor"],
dtype=self.dtype_for('raw_records'))
for fn in sorted(glob.glob(f'{path}/*'))
]
records = np.concatenate(records)
records = strax.sort_by_time(records)
first_start, last_start, last_end = None, None, None
if len(records):
first_start, last_start = records[0]['time'], records[-1]['time']
# Records are sorted by (start)time and are of variable length.
# Their end-times can differ. In the most pessimistic case we have
# to look back one record length for each channel.
tot_channels = np.sum(
[np.diff(x) + 1 for x in self.config['channel_map'].values()])
look_n_samples = self.config["record_length"] * tot_channels
last_end = strax.endtime(records[-look_n_samples:]).max()
if first_start < start or last_start >= end:
raise ValueError(
f"Bad data from DAQ: chunk {path} should contain data "
f"that starts in [{start}, {end}), but we see start times "
f"ranging from {first_start} to {last_start}.")
if kind == 'central':
result = records
break_time = None
else:
# Find a time at which we can safely partition the data.
min_gap = self.config['safe_break_in_pulses']
if not len(records) or last_end + min_gap < end:
# There is enough room at the end of the data
break_time = end - min_gap
result = records if kind == 'post' else records[:0]
else:
# Let's hope there is some quiet time in the middle
try:
result, break_time = strax.from_break(
records,
safe_break=min_gap,
# Records from the last chunk can extend as far as:
not_before=(
start +
self.config['record_length'] * self.dt_max),
left=kind == 'post',
tolerant=False)
except strax.NoBreakFound:
# We still have to break somewhere, but this can involve
# throwing away data.
# Let's do it at the end of the chunk
# TODO: find a better time, e.g. a longish-but-not-quite
# satisfactory gap
break_time = end - min_gap
# Mark the region where data /might/ be removed with
# artificial deadtime.
dead_time_start = (
break_time -
self.config['record_length'] * self.dt_max)
warnings.warn(
f"Data in {path} is so dense that no {min_gap} "
f"ns break exists: data loss inevitable. "
f"Inserting artificial deadtime between "
f"{dead_time_start} and {end}.",
ArtificialDeadtimeInserted)
if kind == 'pre':
# Give the artificial deadtime past the break
result = self._artificial_dead_time(start=break_time,
end=end,
dt=self.dt_max)
else:
# Remove data that would stick out
result = records[strax.endtime(records) <= break_time]
# Add the artificial deadtime until the break
result = strax.sort_by_time(
np.concatenate([
result,
self._artificial_dead_time(
start=dead_time_start,
end=break_time,
dt=self.dt_max)
]))
if self.config['erase']:
shutil.rmtree(path)
return result, break_time
