def compute(self, raw_records_coin_nv): # Do not trust in DAQ + strax.baseline to leave the # out-of-bounds samples to zero. r = strax.raw_to_records(raw_records_coin_nv) del raw_records_coin_nv r = strax.sort_by_time(r) strax.zero_out_of_bounds(r) strax.baseline(r, baseline_samples=self.config['baseline_samples_nv'], flip=True) strax.integrate(r) strax.zero_out_of_bounds(r) hits = strax.find_hits( r, min_amplitude=self.config['hit_min_amplitude_nv']) le, re = self.config['save_outside_hits_nv'] r = strax.cut_outside_hits(r, hits, left_extension=le, right_extension=re) strax.zero_out_of_bounds(r) rlinks = strax.record_links(r) r = clean_up_empty_records(r, rlinks, only_last=True) return r
def compute(self, raw_records): # Convert everything to the records data type -- adds extra fields. records = strax.raw_to_records(raw_records) del raw_records # calculate baseline and baseline rms strax.baseline(records, baseline_samples=self.config['baseline_samples'], flip=True) # find all hits hits = strax.find_hits( records, min_amplitude=self.hit_thresholds, min_height_over_noise=self.config['hit_min_height_over_noise'], ) # sort hits by record_i and time, then find LED hit and afterpulse # hits within the same record hits_ap = find_ap( hits, records, LED_window_left=self.config['LED_window_left'], LED_window_right=self.config['LED_window_right'], hit_left_extension=self.hit_left_extension, hit_right_extension=self.hit_right_extension, ) hits_ap['area_pe'] = hits_ap['area'] * self.to_pe[hits_ap['channel']] hits_ap['height_pe'] = hits_ap['height'] * self.to_pe[ hits_ap['channel']] return hits_ap
def compute(self, raw_records_coin_nv): # Do not trust in DAQ + strax.baseline to leave the # out-of-bounds samples to zero. r = strax.raw_to_records(raw_records_coin_nv) del raw_records_coin_nv r = strax.sort_by_time(r) strax.zero_out_of_bounds(r) strax.baseline(r, baseline_samples=self.baseline_samples, flip=True) if self.config['min_samples_alt_baseline_nv']: m = r['pulse_length'] > self.config['min_samples_alt_baseline_nv'] if np.any(m): # Correcting baseline after PMT saturated signals r[m] = median_baseline(r[m]) strax.integrate(r) strax.zero_out_of_bounds(r) hits = strax.find_hits(r, min_amplitude=self.hit_thresholds) le, re = self.config['save_outside_hits_nv'] r = strax.cut_outside_hits(r, hits, left_extension=le, right_extension=re) strax.zero_out_of_bounds(r) return r
def compute(self, chunk_i): pre, current, post = self._chunk_paths(chunk_i) if chunk_i == 0 and pre: pre = False print(f"There should be no {pre} dir for chunk 0: ignored") records = np.concatenate( ([self._load_chunk(pre, kind='pre')] if pre else []) + [self._load_chunk(current)] + ([self._load_chunk(post, kind='post')] if post else [])) strax.baseline(records) strax.integrate(records) if len(records): # Convert time to time in ns since unix epoch. # Ensure the offset is a whole digitizer sample t0 = int(self.config["run_start_time"] * int(1e9)) dt = records[0]['dt'] t0 = dt * (t0 // dt) records["time"] += t0 timespan_sec = (records[-1]['time'] - records[0]['time']) / 1e9 print(f'{chunk_i}: read {records.nbytes/1e6:.2f} MB ' f'({len(records)} records, ' f'{timespan_sec:.2f} sec) from readers') else: print(f'{chunk_i}: read an empty chunk!') return records
def raw_records_matrix(context, run_id, raw_records, time_range): # Convert raw to records. We may not be able to baseline correctly # at the start of the range due to missing zeroth fragments records = strax.raw_to_records(raw_records) strax.baseline(records, allow_sloppy_chunking=True) return context.records_matrix(run_id=run_id, records=records, time_range=time_range)
def finish_results(): nonlocal results records = np.concatenate(results) # In strax data, records are always stored # sorted, baselined and integrated records = strax.sort_by_time(records) strax.baseline(records) strax.integrate(records) results = [] return records
def compute(self, raw_records_aqmon): rec = strax.raw_to_records(raw_records_aqmon) strax.sort_by_time(rec) strax.zero_out_of_bounds(rec) strax.baseline(rec, baseline_samples=self.config['baseline_samples_aqmon'], flip=True) aqmon_hits = strax.find_hits( rec, min_amplitude=self.config['hit_min_amplitude_aqmon']) return aqmon_hits
def compute(self, raw_records_nv, start, end): strax.zero_out_of_bounds(raw_records_nv) # First we have to split rr into records and lone records: # Please note that we consider everything as a lone record which # does not satisfy the coincidence requirement intervals = coincidence(raw_records_nv, self.config['coincidence_level_recorder_nv'], self.config['resolving_time_recorder_nv']) # Always save the first and last resolving_time nanoseconds (e.g. 600 ns) since we cannot guarantee the gap # size to be larger. (We cannot use an OverlapingWindow plugin either since it requires disjoint objects.) if len(intervals): intervals_with_bounds = np.zeros((len(intervals) + 2, 2), dtype=np.int64) intervals_with_bounds[1:-1, :] = intervals intervals_with_bounds[0, :] = start, min( start + self.config['resolving_time_recorder_nv'], intervals[0, 0]) intervals_with_bounds[-1, :] = max( end - self.config['resolving_time_recorder_nv'], intervals[-1, 1]), end del intervals else: intervals_with_bounds = np.zeros((0, 2), dtype=np.int64) neighbors = strax.record_links(raw_records_nv) mask = pulse_in_interval(raw_records_nv, neighbors, *np.transpose(intervals_with_bounds)) rr, lone_records = straxen.mask_and_not(raw_records_nv, mask) # Compute some properties of the lone_records: # We compute only for lone_records baseline etc. since # raw_records_nv will be deleted, otherwise we could not change # the settings and reprocess the data in case of raw_records_nv lr = strax.raw_to_records(lone_records) del lone_records lr = strax.sort_by_time(lr) strax.zero_out_of_bounds(lr) strax.baseline( lr, baseline_samples=self.config['nbaseline_samples_lone_records_nv'], flip=True) strax.integrate(lr) lrs, lr = compute_lone_records(lr, self.config['channel_map']['nveto'], self.config['n_lone_records_nv']) lrs['time'] = start lrs['endtime'] = end return { 'raw_records_coin_nv': rr, 'lone_raw_records_nv': lr, 'lone_raw_record_statistics_nv': lrs }
def final_results(self, record_j): records = self.record_buffer[:record_j] # Copy the records from buffer records = strax.sort_by_time( records) # Must keep this for sorted output strax.baseline(records) strax.integrate(records) _truth = self.truth_buffer[self.truth_buffer['fill']] # Return truth without 'fill' field truth = np.zeros(len(_truth), dtype=instruction_dtype + truth_extra_dtype) for name in truth.dtype.names: truth[name] = _truth[name] return dict(raw_records=records, truth=truth)
def compute(self, chunk_i): fp = self._chunk_path(chunk_i) records = self._load_chunk(fp) strax.baseline(records) strax.integrate(records) if len(records): timespan_sec = (records[-1]['time'] - records[0]['time']) / 1e9 print(f'{chunk_i}: read {records.nbytes/1e6:.2f} MB ' f'({len(records)} records, ' f'{timespan_sec:.2f} live seconds)') else: print(f'{chunk_i}: read an empty chunk!') return records
def compute(self, chunk_i): pre, current, post = self._chunk_paths(chunk_i) records = np.concatenate( ([self._load_chunk(pre, kind='pre')] if pre else []) + [self._load_chunk(current)] + ([self._load_chunk(post, kind='post')] if post else [])) strax.baseline(records) strax.integrate(records) timespan_sec = (records[-1]['time'] - records[0]['time']) / 1e9 print(f'{chunk_i}: read {records.nbytes/1e6:.2f} MB ' f'({len(records)} records, ' f'{timespan_sec:.2f} sec) from readers') return records
def raw_records_matrix(context, run_id, raw_records, time_range, ignore_max_sample_warning=False, max_samples=DEFAULT_MAX_SAMPLES, **kwargs): # Convert raw to records. We may not be able to baseline correctly # at the start of the range due to missing zeroth fragments records = strax.raw_to_records(raw_records) strax.baseline(records, allow_sloppy_chunking=True) strax.zero_out_of_bounds(records) return context.records_matrix(run_id=run_id, records=records, time_range=time_range, max_samples=max_samples, ignore_max_sample_warning=ignore_max_sample_warning, **kwargs)
def compute(self, raw_records_aqmon): not_allowed_channels = (set(np.unique(raw_records_aqmon['channel'])) - set(self.aqmon_channels)) if not_allowed_channels: raise ValueError( f'Unknown channel {not_allowed_channels}. Only know {self.aqmon_channels}' ) if self.check_raw_record_aqmon_overlaps: straxen.check_overlaps(raw_records_aqmon, n_channels=max(AqmonChannels).value + 1) records = strax.raw_to_records(raw_records_aqmon) strax.zero_out_of_bounds(records) strax.baseline(records, baseline_samples=self.baseline_samples_aqmon, flip=True) aqmon_hits = self.find_aqmon_hits_per_channel(records) aqmon_hits = strax.sort_by_time(aqmon_hits) return aqmon_hits
def compute(self, chunk_i): pre, current, post = self._chunk_paths(chunk_i) if chunk_i == 0 and pre: pre = False print(f"There should be no {pre} dir for chunk 0: ignored") records = np.concatenate( ([self._load_chunk(pre, kind='pre')] if pre else []) + [self._load_chunk(current)] + ([self._load_chunk(post, kind='post')] if post else [])) strax.baseline(records) strax.integrate(records) if len(records): timespan_sec = (records[-1]['time'] - records[0]['time']) / 1e9 print(f'{chunk_i}: read {records.nbytes/1e6:.2f} MB ' f'({len(records)} records, ' f'{timespan_sec:.2f} sec) from readers') else: print(f'{chunk_i}: read an empty chunk!') return records
def compute(self, raw_records_coin_nv): # Do not trust in DAQ + strax.baseline to leave the # out-of-bounds samples to zero. r = strax.raw_to_records(raw_records_coin_nv) del raw_records_coin_nv r = strax.sort_by_time(r) strax.zero_out_of_bounds(r) strax.baseline(r, baseline_samples=self.baseline_samples, flip=True) strax.integrate(r) strax.zero_out_of_bounds(r) hits = strax.find_hits(r, min_amplitude=self.hit_thresholds) le, re = self.config['save_outside_hits_nv'] r = strax.cut_outside_hits(r, hits, left_extension=le, right_extension=re) strax.zero_out_of_bounds(r) return r
def compute(self, raw_records, start, end): if self.config['check_raw_record_overlaps']: check_overlaps(raw_records, n_channels=3000) # Throw away any non-TPC records; this should only happen for XENON1T # converted data raw_records = raw_records[ raw_records['channel'] < self.config['n_tpc_pmts']] # Convert everything to the records data type -- adds extra fields. r = strax.raw_to_records(raw_records) del raw_records # Do not trust in DAQ + strax.baseline to leave the # out-of-bounds samples to zero. # TODO: better to throw an error if something is nonzero strax.zero_out_of_bounds(r) strax.baseline( r, baseline_samples=self.config['baseline_samples'], allow_sloppy_chunking=self.config['allow_sloppy_chunking'], flip=True) strax.integrate(r) pulse_counts = count_pulses(r, self.config['n_tpc_pmts']) pulse_counts['time'] = start pulse_counts['endtime'] = end if len(r) and self.hev_enabled: r, r_vetoed, veto_regions = software_he_veto( r, self.to_pe, end, area_threshold=self.config['tail_veto_threshold'], veto_length=self.config['tail_veto_duration'], veto_res=self.config['tail_veto_resolution'], pass_veto_extend=self.config['tail_veto_pass_extend'], pass_veto_fraction=self.config['tail_veto_pass_fraction'], max_veto_value=self.config['max_veto_value']) # In the future, we'll probably want to sum the waveforms # inside the vetoed regions, so we can still save the "peaks". del r_vetoed else: veto_regions = np.zeros(0, dtype=strax.hit_dtype) if len(r): # Find hits # -- before filtering,since this messes with the with the S/N hits = strax.find_hits(r, min_amplitude=straxen.hit_min_amplitude( self.config['hit_min_amplitude'])) if self.config['pmt_pulse_filter']: # Filter to concentrate the PMT pulses strax.filter_records(r, np.array(self.config['pmt_pulse_filter'])) le, re = self.config['save_outside_hits'] r = strax.cut_outside_hits(r, hits, left_extension=le, right_extension=re) # Probably overkill, but just to be sure... strax.zero_out_of_bounds(r) return dict(records=r, pulse_counts=pulse_counts, veto_regions=veto_regions)
def compute(self, raw_records_nv, start, end): if self.config['check_raw_record_overlaps_nv']: straxen.check_overlaps(raw_records_nv, n_channels=3000) # Cover the case if we do not want to have any coincidence: if self.config['coincidence_level_recorder_nv'] <= 1: rr = raw_records_nv lr = np.zeros(0, dtype=self.dtype['lone_raw_records_nv']) lrs = np.zeros(0, dtype=self.dtype['lone_raw_record_statistics_nv']) return { 'raw_records_coin_nv': rr, 'lone_raw_records_nv': lr, 'lone_raw_record_statistics_nv': lrs } # Search for hits to define coincidence intervals: temp_records = strax.raw_to_records(raw_records_nv) temp_records = strax.sort_by_time(temp_records) strax.zero_out_of_bounds(temp_records) strax.baseline(temp_records, baseline_samples=self.baseline_samples, flip=True) hits = strax.find_hits(temp_records, min_amplitude=self.hit_thresholds) del temp_records # First we have to split rr into records and lone records: # Please note that we consider everything as a lone record which # does not satisfy the coincidence requirement intervals = find_coincidence( hits, self.config['coincidence_level_recorder_nv'], self.config['resolving_time_recorder_nv'], self.config['pre_trigger_time_nv']) del hits # Always save the first and last resolving_time nanoseconds (e.g. 600 ns) since we cannot guarantee the gap # size to be larger. (We cannot use an OverlapingWindow plugin either since it requires disjoint objects.) if len(intervals): intervals_with_bounds = np.zeros(len(intervals) + 2, dtype=strax.time_fields) intervals_with_bounds['time'][1:-1] = intervals['time'] intervals_with_bounds['endtime'][1:-1] = intervals['endtime'] intervals_with_bounds['time'][0] = start intervals_with_bounds['endtime'][0] = min( start + self.config['resolving_time_recorder_nv'], intervals['time'][0]) intervals_with_bounds['time'][-1] = max( end - self.config['resolving_time_recorder_nv'], intervals['endtime'][-1]) intervals_with_bounds['endtime'][-1] = end del intervals else: intervals_with_bounds = np.zeros((0, 2), dtype=strax.time_fields) neighbors = strax.record_links(raw_records_nv) mask = pulse_in_interval( raw_records_nv, neighbors, intervals_with_bounds['time'], intervals_with_bounds['endtime'], ) rr, lone_records = straxen.mask_and_not(raw_records_nv, mask) # Compute some properties of the lone_records: # We compute only for lone_records baseline etc. since # raw_records_nv will be deleted, otherwise we could not change # the settings and reprocess the data in case of raw_records_nv lr = strax.raw_to_records(lone_records) del lone_records lr = strax.sort_by_time(lr) strax.zero_out_of_bounds(lr) strax.baseline(lr, baseline_samples=self.baseline_samples, flip=True) strax.integrate(lr) lrs, lr = compute_lone_records(lr, self.config['channel_map']['nveto'], self.config['n_lone_records_nv']) lrs['time'] = start lrs['endtime'] = end return { 'raw_records_coin_nv': rr, 'lone_raw_records_nv': lr, 'lone_raw_record_statistics_nv': lrs }
def pax_to_records(input_filename, samples_per_record=strax.DEFAULT_RECORD_LENGTH): """Return pulse records array from pax zip input_filename""" from pax import core # Pax is not a dependency mypax = core.Processor( 'XENON1T', config_dict=dict( pax=dict(look_for_config_in_runs_db=False, plugin_group_names=['input'], encoder_plugin=None, input_name=input_filename), # Fast startup: skip loading big maps WaveformSimulator=dict(s1_light_yield_map='placeholder_map.json', s2_light_yield_map='placeholder_map.json', s1_patterns_file=None, s2_patterns_file=None))) def get_events(): for e in mypax.get_events(): yield mypax.process_event(e) # We loop over the events twice for convenience # Yeah, this is probably not optimal pulse_lengths = np.array( [p.length for e in get_events() for p in e.pulses]) n_records = strax.records_needed(pulse_lengths, samples_per_record).sum() records = np.zeros(n_records, dtype=strax.record_dtype(samples_per_record)) output_record_index = 0 # Record offset in data for event in get_events(): for p in event.pulses: n_records = strax.records_needed(p.length, samples_per_record) for rec_i in range(n_records): r = records[output_record_index] r['time'] = (event.start_time + p.left * 10 + rec_i * samples_per_record * 10) r['channel'] = p.channel r['pulse_length'] = p.length r['record_i'] = rec_i r['dt'] = 10 # How much are we storing in this record? if rec_i != n_records - 1: # There's more chunks coming, so we store a full chunk n_store = samples_per_record assert p.length > samples_per_record * (rec_i + 1) else: # Just enough to store the rest of the data # Note it's not p.length % samples_per_record!!! # (that would be zero if we have to store a full record) n_store = p.length - samples_per_record * rec_i assert 0 <= n_store <= samples_per_record r['length'] = n_store offset = rec_i * samples_per_record r['data'][:n_store] = p.raw_data[offset:offset + n_store] output_record_index += 1 mypax.shutdown() # In strax data, records are always stored # sorted, baselined and integrated records = strax.sort_by_time(records) strax.baseline(records) strax.integrate(records) return records
def get_nim_edge(raw_records, threshold=500): records = strax.raw_to_records(raw_records) strax.baseline(records) hits = strax.find_hits(records, min_amplitude=threshold) return hits
def plot_pulses(context, raw_records, run_id, time_range, plot_hits=False, plot_median=False, max_plots=20, store_pdf=False, path='', detector_ending=''): """ Plots nveto pulses for a list of records. :param context: Context to be used. :param plot_hits: If True plot hit boundaries including the left and right extension as orange shaded regions. :param plot_median: If true plots pulses sample median as dotted line. :param max_plots: Limits the number of figures. If you would like to plot more pulses you should put the plots in a PDF. :param store_pdf: If true figures are put to a PDF instead of plotting them to your notebook. The file name is automatically generated including the time range and run_id. :param path: Relative path where the PDF should be stored. By default it is the directory of the notebook. :param detector_ending: Ending of the corresponding detector. Empty string for TPC '_nv' for neutron-veto and '_mv' muon-veto. """ # Register records plugin to get settings p = context.get_single_plugin(run_id, 'records' + detector_ending) # Compute strax baseline and baseline_rms: records = strax.raw_to_records(raw_records) records = strax.sort_by_time(records) strax.zero_out_of_bounds(records) baseline_key = [ key for key in p.config.keys() if 'baseline_samples' in key ][0] if isinstance(p.config[baseline_key], int): baseline_samples = p.config[baseline_key] else: baseline_samples = straxen.get_correction_from_cmt( run_id, p.config[baseline_key]) strax.baseline(records, baseline_samples=baseline_samples, flip=True) nfigs = 1 if store_pdf and time_range is None: raise ValueError(f'Specify time range!') if store_pdf: from matplotlib.backends.backend_pdf import PdfPages fname = f'pulses_{run_id}_{time_range[0]}_{time_range[1]}.pdf' fname = os.path.join(path, fname) pdf = PdfPages(fname) hits = None # needed for delete if false for inds in _yield_pulse_indices(raw_records): # Grouped our pulse so now plot: rr_pulse = raw_records[inds] r_pulse = records[inds] fig, axes = straxen.plot_single_pulse(rr_pulse, run_id) if detector_ending == '_nv': # We only store for the nv digitizer baseline values: axes.axhline(rr_pulse[0]['baseline'], ls='dashed', color='k', label=f'D. Bas.: {rr_pulse[0]["baseline"]} ADC') baseline = r_pulse[0]['baseline'] baseline_rms = r_pulse[0]['baseline_rms'] axes.axhline( baseline, ls='solid', color='k', label= f'Strax Bas. +/-RMS:\n ({baseline:.2f}+/-{baseline_rms:.2f}) ADC') xlim = axes.get_xlim() axes.fill_between(xlim, [baseline + baseline_rms] * 2, [baseline - baseline_rms] * 2, color='gray', alpha=0.4) # check type of p.hit_thresholds if isinstance(p.hit_thresholds, int): thr = p.hit_thresholds elif isinstance(p.hit_thresholds, np.ndarray): thr = p.hit_thresholds[rr_pulse['channel']][0] if plot_median: # Plot median if asked. # Have to make pulse again: pulse = straxen.matplotlib_utils._make_pulse(rr_pulse) median = np.median(pulse) axes.axhline(median, ls='dotted', color='k', label=f'Median Bas.: {median:.0f} ADC') axes.axhline(median - thr, ls='dotted', color='orange') if plot_hits: min_amplitude = thr axes.axhline(baseline - min_amplitude, color='orange', label='Hitfinder threshold') hits = strax.find_hits(r_pulse, min_amplitude=min_amplitude) if detector_ending != '_he': # We don't have 'save_outside_hits_he' at all! le, re = p.config['save_outside_hits' + detector_ending] else: le, re = p.config['save_outside_hits'] start = (hits['time'] - r_pulse[0]['time']) / r_pulse[0]['dt'] - le end = (strax.endtime(hits) - r_pulse[0]['time']) / r_pulse[0]['dt'] + re ylim = axes.get_ylim() for s, e in zip(start, end): plt.fill_between((s, e), *ylim, alpha=0.2, color='orange') axes.set_ylim(*ylim) plt.legend() axes.set_xlim(*xlim) if store_pdf: plt.close() pdf.savefig(fig) nfigs += 1 if max_plots is not None and nfigs > max_plots: break if store_pdf: pdf.close() del records, hits