def calc_local_tc(event_generator, calib, pixel, gain): all_slopes = [[] for i in range(1024)] stop_cells = np.zeros(1024, dtype=int) for event in tqdm(event_generator): event = calib(event) calibrated = event.data[pixel][gain] zero_crossings = np.where(np.diff(np.signbit(calibrated)))[0] slopes = calibrated[zero_crossings + 1] - calibrated[zero_crossings] absolute_slopes = np.abs(slopes) sc = event.header.stop_cells[pixel][gain] stop_cells[sc % 1024] += 1 zero_crossing_cells = dr.sample2cell(zero_crossings + 1, stop_cell=sc, total_cells=1024) for abs_slope, cell_id in zip(absolute_slopes, zero_crossing_cells): all_slopes[cell_id].append(abs_slope) tc = pd.DataFrame({ "cell_width_mean": np.zeros(len(all_slopes), dtype=np.float32), "cell_width_std": np.zeros(len(all_slopes), dtype=np.float32), "number_of_crossings": np.zeros(len(all_slopes), dtype=np.int32), "stop_cell": stop_cells, "slope_mean": np.zeros(len(all_slopes), dtype=np.float32), }) for cell_id, slopes in enumerate(all_slopes): slopes = np.array(slopes) tc.loc[cell_id, "number_of_crossings"] = len(slopes) tc.loc[cell_id, "cell_width_mean"] = np.mean(slopes) # np.median(slopes) tc.loc[cell_id, "slope_mean"] = slopes.mean() if False: plt.hist(slopes, bins=50, histtype="step") plt.title(str(len(slopes))) ax = plt.gca() ax.ticklabel_format(useOffset=False) plt.savefig("slopes/{0:04d}.png".format(cell_id)) plt.close("all") tc.loc[cell_id, "cell_width_std"] = slopes.std() / np.sqrt(len(slopes)) average_of_all_slopes = tc.cell_width_mean.dropna().mean() tc["cell_width_mean"] /= average_of_all_slopes tc["cell_width_std"] /= average_of_all_slopes return tc
def calc_local_tc(event_generator, calib, pixel, gain): all_slopes = [[] for i in range(1024)] stop_cells = np.zeros(1024, dtype=int) for event in tqdm(event_generator): event = calib(event) calibrated = event.data[pixel][gain] zero_crossings = np.where(np.diff(np.signbit(calibrated)))[0] slopes = calibrated[zero_crossings + 1] - calibrated[zero_crossings] absolute_slopes = np.abs(slopes) sc = event.header.stop_cells[pixel][gain] stop_cells[sc%1024] += 1 zero_crossing_cells = dr.sample2cell(zero_crossings+1, stop_cell=sc, total_cells=1024) for abs_slope, cell_id in zip(absolute_slopes, zero_crossing_cells): all_slopes[cell_id].append(abs_slope) tc = pd.DataFrame({ "cell_width_mean": np.zeros(len(all_slopes), dtype=np.float32), "cell_width_std": np.zeros(len(all_slopes), dtype=np.float32), "number_of_crossings": np.zeros(len(all_slopes), dtype=np.int32), "stop_cell": stop_cells, "slope_mean": np.zeros(len(all_slopes), dtype=np.float32), }) for cell_id, slopes in enumerate(all_slopes): slopes = np.array(slopes) tc.loc[cell_id, "number_of_crossings"] = len(slopes) tc.loc[cell_id, "cell_width_mean"] = np.mean(slopes) # np.median(slopes) tc.loc[cell_id, "slope_mean"] = slopes.mean() if False: plt.hist(slopes, bins=50, histtype="step") plt.title(str(len(slopes))) ax = plt.gca() ax.ticklabel_format(useOffset=False) plt.savefig("slopes/{0:04d}.png".format(cell_id)) plt.close("all") tc.loc[cell_id, "cell_width_std"] = slopes.std() / np.sqrt(len(slopes)) average_of_all_slopes = tc.cell_width_mean.dropna().mean() tc["cell_width_mean"] /= average_of_all_slopes tc["cell_width_std"] /= average_of_all_slopes return tc
arrival_time = np.zeros(NN, dtype='f4') arrival_time_no_calib = np.zeros(NN, dtype='f4') trapz = np.zeros(NN, dtype='f4') simps = np.zeros(NN, dtype='f4') for i, event in enumerate(progress_bar(run, leave=True)): raw_data = event.data[ch][gain] stop_cell = event.header.stop_cells[ch][gain] calibrated = raw_data - offset[stop_cell:stop_cell+run.roi] t = cell_width[stop_cell:stop_cell+run.roi].cumsum() max_pos[i] = np.argmax(calibrated) s = slice(max_pos[i]-half_integration_window, max_pos[i]+half_integration_window+1) samples = np.arange(s.start, s.stop) cells = dr.sample2cell(samples, stop_cell, total_cells=1024) DLE = partial(digital_leading_edge_discriminator, data=calibrated, threshold=1000) arrival_time[i] = DLE(time=t) arrival_time_no_calib[i] = DLE(time=np.arange(len(calibrated))) integral[i] = calibrated[s].sum() integral_weighted[i] = (calibrated[s] * midpoint_width[cells]).sum() trapz[i] = np.trapz(calibrated[s], t[s]) simps[i] = scipy.integrate.simps(calibrated[s], t[s]) df = pd.DataFrame({ "integral": integral, "integral_weighted": integral_weighted, "max_pos": max_pos, "arrival_time": arrival_time,
gain = args["--gain"] assert gain in ["high", "low"] all_slopes = [[] for i in range(1024)] #fig, a = plt.subplots(1) for event in tqdm(event_generator): event = calib(event) calibrated = event.data[pixel][gain] zero_crossings = np.where(np.diff(np.signbit(calibrated)))[0] slope = calibrated[zero_crossings + 1] - calibrated[zero_crossings] zero_crossing_cells = dr.sample2cell( zero_crossings, stop_cell=event.header.stop_cells[pixel][gain], total_cells=1024) for crossing_number, cell_id in enumerate(zero_crossing_cells): all_slopes[cell_id].append(np.abs(slope[crossing_number])) slope_mean = np.zeros(1024) slope_std = np.zeros(1024) for cell_id, slopes in enumerate(all_slopes): slopes = np.array(slopes) slope_mean[cell_id] = slopes.mean() slope_std[cell_id] = slopes.std() / np.sqrt(len(slopes)) print(slope_std) df = pd.DataFrame({ "cell_width_mean": slope_mean / slope_mean.mean(),
def main(): args = docopt(__doc__) args["--max_events"] = None if args["--max_events"] is None else int(args["--max_events"]) args["--skip_begin"] = int(args["--skip_begin"]) args["--skip_end"] = int(args["--skip_end"]) args["--do_channel8"] = bool(args["--do_channel8"]) print(args['<outputfile>']) if os.path.isfile(args['<outputfile>']): answer = input('Outputfile {} exists. Do you want to overwrite? y/[n] '.format(args['<outputfile>'])) if not answer.lower().startswith('y'): sys.exit() store = pd.HDFStore(args['<outputfile>'], 'w') adc = {} delta_t = {} for pixel in range(8 if args["--do_channel8"] else 7): for gain in ["high", "low"]: adc[pixel, gain] = [array('H') for i in range(4096)] delta_t[pixel, gain] = [array('f') for i in range(4096)] if not args["--do_channel8"]: # We need to put nans for channel 8 into the output file, since the # rest of the system expects this data to be there ... even if it its nan. result = pd.DataFrame( np.full((4096, 4), np.nan), columns=['a', 'b', 'c', 'chisq_ndf'] ) result['pixel'] = 7 result['channel'] = "high" result['cell'] = np.arange(4096) store.append('data', result, min_itemsize={'channel': 4}) result['channel'] = "low" store.append('data', result, min_itemsize={'channel': 4}) print("reading raw file(s) into memory:") for eg in [dr.EventGenerator(filename, max_events=args["--max_events"]) for filename in args["<inputfiles>"]]: sample_ids = np.arange(eg.roi) for event in tqdm( iterable=eg, desc=os.path.basename(eg.path), leave=True, unit=' events', ): for pixel, gain in sorted(adc.keys()): sc = event.header.stop_cells[pixel][gain] skip_slice = slice(args["--skip_begin"], -args["--skip_end"]) cell_ids = dr.sample2cell(sample_ids, sc)[skip_slice] data = event.data[pixel][gain][skip_slice] delta_ts = event.time_since_last_readout[pixel][gain][skip_slice] for i, cid in enumerate(cell_ids): delta_t[pixel, gain][cid].append(delta_ts[i]) adc[pixel, gain][cid].append(data[i]) for key in sorted(adc.keys()): # adc and delta_t have the same keys for i in tqdm(range(4096), desc=str(key)): adc[key][i] = np.array(adc[key][i], dtype=adc[key][i].typecode) delta_t[key][i] = np.array(delta_t[key][i], dtype=delta_t[key][i].typecode) adc[key][i] = adc[key][i][~np.isnan(delta_t[key][i])] delta_t[key][i] = delta_t[key][i][~np.isnan(delta_t[key][i])] print("fitting") pool = Parallel(max(psutil.cpu_count()-1, 1)) for key in tqdm(iterable=sorted(adc.keys()), leave=True): result = pd.DataFrame( pool(delayed(fit)(adc[key][i], delta_t[key][i], i) for i in range(4096)), columns=['a', 'b', 'c', 'chisq_ndf'] ) pixel, channel = key result['pixel'] = pixel result['channel'] = channel result['cell'] = np.arange(4096) store.append('data', result, min_itemsize={'channel': 4})
def extract_data( inputfiles, outpath, memory, calibpath=None, extrapath=None, a=None, b=None, ): ''' calculate time lapse dependence for a given capacitor If calib path and/or extrapath are given calibrated data is stored Possible combinations: calibpath: TimelapseCalibration extrapath: MedianTimelapseExtraOffsets calibpath and extrapath: TimelapseCalibrationExtraOffsets ''' if extrapath and not calibpath: print('using: MedianTimelapseExtraOffsets') calib = MedianTimelapseExtraOffsets(extrapath) elif extrapath and calibpath: print('using: TimelapseCalibrationExtraOffsets') calib = TimelapseCalibrationExtraOffsets(calibpath, extrapath) elif not extrapath and calibpath: print('using: TimelapseCalibration') calib = TimelapseCalibration(calibpath) else: print('using: --') calib = NoCalibration() p = psutil.Process(os.getpid()) with pd.HDFStore(outpath, mode='w', comp_level=5, comp_lib='blosc') as store: sample_ids = None for filename in sorted(inputfiles): data = defaultdict(lambda: defaultdict(list)) for event in tqdm( iterable=dr.EventGenerator(filename), desc=os.path.basename(filename), leave=True, unit=' events', ): event = calib(event) if sample_ids is None or sample_ids.shape != event.roi: sample_ids = np.arange(event.roi) for pixel, pixel_data in enumerate(event.data): for gain in pixel_data.dtype.names: stop_cell = event.header.stop_cells[pixel][gain] delta_ts = event.time_since_last_readout[pixel][gain] valid = np.logical_not(np.isnan(delta_ts)) if not np.any(valid): continue data[(pixel, gain)]['delta_t'].extend(delta_ts[valid]) data[(pixel, gain)]['cell'].extend( dr.sample2cell(sample_ids[valid], stop_cell) ) data[(pixel, gain)]['sample'].extend(sample_ids[valid]) data[(pixel, gain)]['adc_counts'].extend(pixel_data[gain][valid]) if p.memory_percent() > memory: write(store, data) data = defaultdict(lambda: defaultdict(list)) write(store, data)
assert gain in ["high", "low"] all_slopes = [[] for i in range(1024)] #fig, a = plt.subplots(1) for event in tqdm(event_generator): event = calib(event) calibrated = event.data[pixel][gain] zero_crossings = np.where(np.diff(np.signbit(calibrated)))[0] slope = calibrated[zero_crossings + 1] - calibrated[zero_crossings] zero_crossing_cells = dr.sample2cell(zero_crossings, stop_cell=event.header.stop_cells[pixel][gain], total_cells=1024) for crossing_number, cell_id in enumerate(zero_crossing_cells): all_slopes[cell_id].append(np.abs(slope[crossing_number])) slope_mean = np.zeros(1024) slope_std = np.zeros(1024) for cell_id, slopes in enumerate(all_slopes): slopes = np.array(slopes) slope_mean[cell_id] = slopes.mean() slope_std[cell_id] = slopes.std() / np.sqrt(len(slopes)) print(slope_std) df = pd.DataFrame({ "cell_width_mean": slope_mean / slope_mean.mean(),
arrival_time_no_calib = np.zeros(NN, dtype='f4') trapz = np.zeros(NN, dtype='f4') simps = np.zeros(NN, dtype='f4') for i, event in enumerate(progress_bar(run, leave=True)): raw_data = event.data[ch][gain] stop_cell = event.header.stop_cells[ch][gain] calibrated = raw_data - offset[stop_cell:stop_cell + run.roi] t = cell_width[stop_cell:stop_cell + run.roi].cumsum() max_pos[i] = np.argmax(calibrated) s = slice(max_pos[i] - half_integration_window, max_pos[i] + half_integration_window + 1) samples = np.arange(s.start, s.stop) cells = dr.sample2cell(samples, stop_cell, total_cells=1024) DLE = partial(digital_leading_edge_discriminator, data=calibrated, threshold=1000) arrival_time[i] = DLE(time=t) arrival_time_no_calib[i] = DLE(time=np.arange(len(calibrated))) integral[i] = calibrated[s].sum() integral_weighted[i] = (calibrated[s] * midpoint_width[cells]).sum() trapz[i] = np.trapz(calibrated[s], t[s]) simps[i] = scipy.integrate.simps(calibrated[s], t[s]) df = pd.DataFrame({ "integral": integral, "integral_weighted": integral_weighted, "max_pos": max_pos,
def main(): args = docopt(__doc__) args["--max_events"] = None if args["--max_events"] is None else int( args["--max_events"]) args["--skip_begin"] = int(args["--skip_begin"]) args["--skip_end"] = int(args["--skip_end"]) args["--do_channel8"] = bool(args["--do_channel8"]) print(args['<outputfile>']) if os.path.isfile(args['<outputfile>']): answer = input( 'Outputfile {} exists. Do you want to overwrite? y/[n] '.format( args['<outputfile>'])) if not answer.lower().startswith('y'): sys.exit() store = pd.HDFStore(args['<outputfile>'], 'w') adc = {} delta_t = {} for pixel in range(8 if args["--do_channel8"] else 7): for gain in ["high", "low"]: adc[pixel, gain] = [array('H') for i in range(4096)] delta_t[pixel, gain] = [array('f') for i in range(4096)] if not args["--do_channel8"]: # We need to put nans for channel 8 into the output file, since the # rest of the system expects this data to be there ... even if it its nan. result = pd.DataFrame(np.full((4096, 4), np.nan), columns=['a', 'b', 'c', 'chisq_ndf']) result['pixel'] = 7 result['channel'] = "high" result['cell'] = np.arange(4096) store.append('data', result, min_itemsize={'channel': 4}) result['channel'] = "low" store.append('data', result, min_itemsize={'channel': 4}) print("reading raw file(s) into memory:") for eg in [ dr.EventGenerator(filename, max_events=args["--max_events"]) for filename in args["<inputfiles>"] ]: sample_ids = np.arange(eg.roi) for event in tqdm( iterable=eg, desc=os.path.basename(eg.path), leave=True, unit=' events', ): for pixel, gain in sorted(adc.keys()): sc = event.header.stop_cells[pixel][gain] skip_slice = slice(args["--skip_begin"], -args["--skip_end"]) cell_ids = dr.sample2cell(sample_ids, sc)[skip_slice] data = event.data[pixel][gain][skip_slice] delta_ts = event.time_since_last_readout[pixel][gain][ skip_slice] for i, cid in enumerate(cell_ids): delta_t[pixel, gain][cid].append(delta_ts[i]) adc[pixel, gain][cid].append(data[i]) for key in sorted(adc.keys()): # adc and delta_t have the same keys for i in tqdm(range(4096), desc=str(key)): adc[key][i] = np.array(adc[key][i], dtype=adc[key][i].typecode) delta_t[key][i] = np.array(delta_t[key][i], dtype=delta_t[key][i].typecode) adc[key][i] = adc[key][i][~np.isnan(delta_t[key][i])] delta_t[key][i] = delta_t[key][i][~np.isnan(delta_t[key][i])] print("fitting") pool = Parallel(max(psutil.cpu_count() - 1, 1)) for key in tqdm(iterable=sorted(adc.keys()), leave=True): result = pd.DataFrame(pool( delayed(fit)(adc[key][i], delta_t[key][i], i) for i in range(4096)), columns=['a', 'b', 'c', 'chisq_ndf']) pixel, channel = key result['pixel'] = pixel result['channel'] = channel result['cell'] = np.arange(4096) store.append('data', result, min_itemsize={'channel': 4})