def test_from_break(records): window = 5 def has_break(x): if len(x) < 2: return False return np.diff(x['time']).max() > window try: left = strax.from_break(records, safe_break=window, left=True, tolerant=False) right = strax.from_break(records, safe_break=window, left=False, tolerant=False) except strax.NoBreakFound: assert not has_break(records) else: assert len(left) + len(right) == len(records) if len(records) > 0: np.testing.assert_equal(np.concatenate([left, right]), records) if len(left) and len(right): assert left[-1]['time'] <= right[0]['time'] - window assert not has_break(right)
def test_from_break(records): if not len(records): return window = 5 def has_break(x): if len(x) < 2: return False for i in range(1, len(x)): if strax.endtime(x[:i]).max() + window <= x[i]['time']: return True return False try: left, t_break_l = strax.from_break(records, safe_break=window, left=True, tolerant=False) right, t_break_r = strax.from_break(records, safe_break=window, left=False, tolerant=False) except strax.NoBreakFound: assert not has_break(records) else: assert t_break_l == t_break_r, "Inconsistent break time" t_break = t_break_l assert has_break(records), f"Found nonexistent break at {t_break}" assert len(left) + len(right) == len(records), "Data loss" if len(records) > 0: np.testing.assert_equal(np.concatenate([left, right]), records) if len(right) and len(left): assert t_break == right[0]['time'] assert strax.endtime(left).max() <= right[0]['time'] - window assert not has_break(right)
def _load_chunk(self, path, kind='central'): records = [ strax.load_file(fn, compressor='blosc', dtype=strax.record_dtype()) for fn in glob.glob(f'{path}/reader_*') ] records = np.concatenate(records) records = strax.sort_by_time(records) if kind == 'central': return records result = strax.from_break( records, safe_break=int(1e3), # TODO config? left=kind == 'post', tolerant=True) if self.config['erase']: shutil.rmtree(path) return result
def _load_chunk(self, path, kind='central'): records = [ strax.load_file(fn, compressor='blosc', dtype=strax.record_dtype()) for fn in sorted(glob.glob(f'{path}/*')) ] records = np.concatenate(records) records = strax.sort_by_time(records) if kind == 'central': result = records else: result = strax.from_break( records, safe_break=self.config['safe_break_in_pulses'], left=kind == 'post', tolerant=True) if self.config['erase']: shutil.rmtree(path) return result
def load_chunk(self, folder, kind='central'): records = np.concatenate([ strax.load_file(os.path.join(folder, f), compressor='blosc', dtype=strax.record_dtype()) for f in os.listdir(folder) ]) records = strax.sort_by_time(records) if kind == 'central': result = records else: if self.config['do_breaks']: result = strax.from_break(records, safe_break=self.config['safe_break'], left=kind == 'post', tolerant=True) else: result = records result['time'] += self.config['run_start'] return result
def _load_chunk(self, path, start, end, kind='central'): records = [ strax.load_file(fn, compressor=self.config["daq_compressor"], dtype=self.dtype_for('raw_records')) for fn in sorted(glob.glob(f'{path}/*')) ] records = np.concatenate(records) records = strax.sort_by_time(records) first_start, last_start, last_end = None, None, None if len(records): first_start, last_start = records[0]['time'], records[-1]['time'] # Records are sorted by (start)time and are of variable length. # Their end-times can differ. In the most pessimistic case we have # to look back one record length for each channel. tot_channels = np.sum( [np.diff(x) + 1 for x in self.config['channel_map'].values()]) look_n_samples = self.config["record_length"] * tot_channels last_end = strax.endtime(records[-look_n_samples:]).max() if first_start < start or last_start >= end: raise ValueError( f"Bad data from DAQ: chunk {path} should contain data " f"that starts in [{start}, {end}), but we see start times " f"ranging from {first_start} to {last_start}.") if kind == 'central': result = records break_time = None else: # Find a time at which we can safely partition the data. min_gap = self.config['safe_break_in_pulses'] if not len(records) or last_end + min_gap < end: # There is enough room at the end of the data break_time = end - min_gap result = records if kind == 'post' else records[:0] else: # Let's hope there is some quiet time in the middle try: result, break_time = strax.from_break( records, safe_break=min_gap, # Records from the last chunk can extend as far as: not_before=( start + self.config['record_length'] * self.dt_max), left=kind == 'post', tolerant=False) except strax.NoBreakFound: # We still have to break somewhere, but this can involve # throwing away data. # Let's do it at the end of the chunk # TODO: find a better time, e.g. a longish-but-not-quite # satisfactory gap break_time = end - min_gap # Mark the region where data /might/ be removed with # artificial deadtime. dead_time_start = ( break_time - self.config['record_length'] * self.dt_max) warnings.warn( f"Data in {path} is so dense that no {min_gap} " f"ns break exists: data loss inevitable. " f"Inserting artificial deadtime between " f"{dead_time_start} and {end}.", ArtificialDeadtimeInserted) if kind == 'pre': # Give the artificial deadtime past the break result = self._artificial_dead_time(start=break_time, end=end, dt=self.dt_max) else: # Remove data that would stick out result = records[strax.endtime(records) <= break_time] # Add the artificial deadtime until the break result = strax.sort_by_time( np.concatenate([ result, self._artificial_dead_time( start=dead_time_start, end=break_time, dt=self.dt_max) ])) if self.config['erase']: shutil.rmtree(path) return result, break_time