def calc_local_tc(event_generator, calib, pixel, gain):
all_slopes = [[] for i in range(1024)]
stop_cells = np.zeros(1024, dtype=int)
for event in tqdm(event_generator):
event = calib(event)
calibrated = event.data[pixel][gain]
zero_crossings = np.where(np.diff(np.signbit(calibrated)))[0]
slopes = calibrated[zero_crossings + 1] - calibrated[zero_crossings]
absolute_slopes = np.abs(slopes)
sc = event.header.stop_cells[pixel][gain]
stop_cells[sc % 1024] += 1
zero_crossing_cells = dr.sample2cell(zero_crossings + 1,
stop_cell=sc,
total_cells=1024)
for abs_slope, cell_id in zip(absolute_slopes, zero_crossing_cells):
all_slopes[cell_id].append(abs_slope)
tc = pd.DataFrame({
"cell_width_mean":
np.zeros(len(all_slopes), dtype=np.float32),
"cell_width_std":
np.zeros(len(all_slopes), dtype=np.float32),
"number_of_crossings":
np.zeros(len(all_slopes), dtype=np.int32),
"stop_cell":
stop_cells,
"slope_mean":
np.zeros(len(all_slopes), dtype=np.float32),
})
for cell_id, slopes in enumerate(all_slopes):
slopes = np.array(slopes)
tc.loc[cell_id, "number_of_crossings"] = len(slopes)
tc.loc[cell_id,
"cell_width_mean"] = np.mean(slopes) # np.median(slopes)
tc.loc[cell_id, "slope_mean"] = slopes.mean()
if False:
plt.hist(slopes, bins=50, histtype="step")
plt.title(str(len(slopes)))
ax = plt.gca()
ax.ticklabel_format(useOffset=False)
plt.savefig("slopes/{0:04d}.png".format(cell_id))
plt.close("all")
tc.loc[cell_id, "cell_width_std"] = slopes.std() / np.sqrt(len(slopes))
average_of_all_slopes = tc.cell_width_mean.dropna().mean()
tc["cell_width_mean"] /= average_of_all_slopes
tc["cell_width_std"] /= average_of_all_slopes
return tc
def calc_local_tc(event_generator, calib, pixel, gain):
all_slopes = [[] for i in range(1024)]
stop_cells = np.zeros(1024, dtype=int)
for event in tqdm(event_generator):
event = calib(event)
calibrated = event.data[pixel][gain]
zero_crossings = np.where(np.diff(np.signbit(calibrated)))[0]
slopes = calibrated[zero_crossings + 1] - calibrated[zero_crossings]
absolute_slopes = np.abs(slopes)
sc = event.header.stop_cells[pixel][gain]
stop_cells[sc%1024] += 1
zero_crossing_cells = dr.sample2cell(zero_crossings+1,
stop_cell=sc,
total_cells=1024)
for abs_slope, cell_id in zip(absolute_slopes, zero_crossing_cells):
all_slopes[cell_id].append(abs_slope)
tc = pd.DataFrame({
"cell_width_mean": np.zeros(len(all_slopes), dtype=np.float32),
"cell_width_std": np.zeros(len(all_slopes), dtype=np.float32),
"number_of_crossings": np.zeros(len(all_slopes), dtype=np.int32),
"stop_cell": stop_cells,
"slope_mean": np.zeros(len(all_slopes), dtype=np.float32),
})
for cell_id, slopes in enumerate(all_slopes):
slopes = np.array(slopes)
tc.loc[cell_id, "number_of_crossings"] = len(slopes)
tc.loc[cell_id, "cell_width_mean"] = np.mean(slopes) # np.median(slopes)
tc.loc[cell_id, "slope_mean"] = slopes.mean()
if False:
plt.hist(slopes, bins=50, histtype="step")
plt.title(str(len(slopes)))
ax = plt.gca()
ax.ticklabel_format(useOffset=False)
plt.savefig("slopes/{0:04d}.png".format(cell_id))
plt.close("all")
tc.loc[cell_id, "cell_width_std"] = slopes.std() / np.sqrt(len(slopes))
average_of_all_slopes = tc.cell_width_mean.dropna().mean()
tc["cell_width_mean"] /= average_of_all_slopes
tc["cell_width_std"] /= average_of_all_slopes
return tc
arrival_time = np.zeros(NN, dtype='f4')
arrival_time_no_calib = np.zeros(NN, dtype='f4')
trapz = np.zeros(NN, dtype='f4')
simps = np.zeros(NN, dtype='f4')
for i, event in enumerate(progress_bar(run, leave=True)):
raw_data = event.data[ch][gain]
stop_cell = event.header.stop_cells[ch][gain]
calibrated = raw_data - offset[stop_cell:stop_cell+run.roi]
t = cell_width[stop_cell:stop_cell+run.roi].cumsum()
max_pos[i] = np.argmax(calibrated)
s = slice(max_pos[i]-half_integration_window, max_pos[i]+half_integration_window+1)
samples = np.arange(s.start, s.stop)
cells = dr.sample2cell(samples, stop_cell, total_cells=1024)
DLE = partial(digital_leading_edge_discriminator, data=calibrated, threshold=1000)
arrival_time[i] = DLE(time=t)
arrival_time_no_calib[i] = DLE(time=np.arange(len(calibrated)))
integral[i] = calibrated[s].sum()
integral_weighted[i] = (calibrated[s] * midpoint_width[cells]).sum()
trapz[i] = np.trapz(calibrated[s], t[s])
simps[i] = scipy.integrate.simps(calibrated[s], t[s])
df = pd.DataFrame({
"integral": integral,
"integral_weighted": integral_weighted,
"max_pos": max_pos,
"arrival_time": arrival_time,
gain = args["--gain"]
assert gain in ["high", "low"]
all_slopes = [[] for i in range(1024)]
#fig, a = plt.subplots(1)
for event in tqdm(event_generator):
event = calib(event)
calibrated = event.data[pixel][gain]
zero_crossings = np.where(np.diff(np.signbit(calibrated)))[0]
slope = calibrated[zero_crossings + 1] - calibrated[zero_crossings]
zero_crossing_cells = dr.sample2cell(
zero_crossings,
stop_cell=event.header.stop_cells[pixel][gain],
total_cells=1024)
for crossing_number, cell_id in enumerate(zero_crossing_cells):
all_slopes[cell_id].append(np.abs(slope[crossing_number]))
slope_mean = np.zeros(1024)
slope_std = np.zeros(1024)
for cell_id, slopes in enumerate(all_slopes):
slopes = np.array(slopes)
slope_mean[cell_id] = slopes.mean()
slope_std[cell_id] = slopes.std() / np.sqrt(len(slopes))
print(slope_std)
df = pd.DataFrame({
"cell_width_mean": slope_mean / slope_mean.mean(),
def main():
args = docopt(__doc__)
args["--max_events"] = None if args["--max_events"] is None else int(args["--max_events"])
args["--skip_begin"] = int(args["--skip_begin"])
args["--skip_end"] = int(args["--skip_end"])
args["--do_channel8"] = bool(args["--do_channel8"])
print(args['<outputfile>'])
if os.path.isfile(args['<outputfile>']):
answer = input('Outputfile {} exists. Do you want to overwrite? y/[n] '.format(args['<outputfile>']))
if not answer.lower().startswith('y'):
sys.exit()
store = pd.HDFStore(args['<outputfile>'], 'w')
adc = {}
delta_t = {}
for pixel in range(8 if args["--do_channel8"] else 7):
for gain in ["high", "low"]:
adc[pixel, gain] = [array('H') for i in range(4096)]
delta_t[pixel, gain] = [array('f') for i in range(4096)]
if not args["--do_channel8"]:
# We need to put nans for channel 8 into the output file, since the
# rest of the system expects this data to be there ... even if it its nan.
result = pd.DataFrame( np.full((4096, 4), np.nan),
columns=['a', 'b', 'c', 'chisq_ndf']
)
result['pixel'] = 7
result['channel'] = "high"
result['cell'] = np.arange(4096)
store.append('data', result, min_itemsize={'channel': 4})
result['channel'] = "low"
store.append('data', result, min_itemsize={'channel': 4})
print("reading raw file(s) into memory:")
for eg in [dr.EventGenerator(filename, max_events=args["--max_events"]) for filename in args["<inputfiles>"]]:
sample_ids = np.arange(eg.roi)
for event in tqdm(
iterable=eg,
desc=os.path.basename(eg.path),
leave=True,
unit=' events',
):
for pixel, gain in sorted(adc.keys()):
sc = event.header.stop_cells[pixel][gain]
skip_slice = slice(args["--skip_begin"], -args["--skip_end"])
cell_ids = dr.sample2cell(sample_ids, sc)[skip_slice]
data = event.data[pixel][gain][skip_slice]
delta_ts = event.time_since_last_readout[pixel][gain][skip_slice]
for i, cid in enumerate(cell_ids):
delta_t[pixel, gain][cid].append(delta_ts[i])
adc[pixel, gain][cid].append(data[i])
for key in sorted(adc.keys()):
# adc and delta_t have the same keys
for i in tqdm(range(4096), desc=str(key)):
adc[key][i] = np.array(adc[key][i], dtype=adc[key][i].typecode)
delta_t[key][i] = np.array(delta_t[key][i], dtype=delta_t[key][i].typecode)
adc[key][i] = adc[key][i][~np.isnan(delta_t[key][i])]
delta_t[key][i] = delta_t[key][i][~np.isnan(delta_t[key][i])]
print("fitting")
pool = Parallel(max(psutil.cpu_count()-1, 1))
for key in tqdm(iterable=sorted(adc.keys()), leave=True):
result = pd.DataFrame(
pool(delayed(fit)(adc[key][i], delta_t[key][i], i) for i in range(4096)),
columns=['a', 'b', 'c', 'chisq_ndf']
)
pixel, channel = key
result['pixel'] = pixel
result['channel'] = channel
result['cell'] = np.arange(4096)
store.append('data', result, min_itemsize={'channel': 4})
def extract_data(
inputfiles,
outpath,
memory,
calibpath=None,
extrapath=None,
a=None,
b=None,
):
'''
calculate time lapse dependence for a given capacitor
If calib path and/or extrapath are given calibrated data is stored
Possible combinations:
calibpath: TimelapseCalibration
extrapath: MedianTimelapseExtraOffsets
calibpath and extrapath: TimelapseCalibrationExtraOffsets
'''
if extrapath and not calibpath:
print('using: MedianTimelapseExtraOffsets')
calib = MedianTimelapseExtraOffsets(extrapath)
elif extrapath and calibpath:
print('using: TimelapseCalibrationExtraOffsets')
calib = TimelapseCalibrationExtraOffsets(calibpath, extrapath)
elif not extrapath and calibpath:
print('using: TimelapseCalibration')
calib = TimelapseCalibration(calibpath)
else:
print('using: --')
calib = NoCalibration()
p = psutil.Process(os.getpid())
with pd.HDFStore(outpath, mode='w', comp_level=5, comp_lib='blosc') as store:
sample_ids = None
for filename in sorted(inputfiles):
data = defaultdict(lambda: defaultdict(list))
for event in tqdm(
iterable=dr.EventGenerator(filename),
desc=os.path.basename(filename),
leave=True,
unit=' events',
):
event = calib(event)
if sample_ids is None or sample_ids.shape != event.roi:
sample_ids = np.arange(event.roi)
for pixel, pixel_data in enumerate(event.data):
for gain in pixel_data.dtype.names:
stop_cell = event.header.stop_cells[pixel][gain]
delta_ts = event.time_since_last_readout[pixel][gain]
valid = np.logical_not(np.isnan(delta_ts))
if not np.any(valid):
continue
data[(pixel, gain)]['delta_t'].extend(delta_ts[valid])
data[(pixel, gain)]['cell'].extend(
dr.sample2cell(sample_ids[valid], stop_cell)
)
data[(pixel, gain)]['sample'].extend(sample_ids[valid])
data[(pixel, gain)]['adc_counts'].extend(pixel_data[gain][valid])
if p.memory_percent() > memory:
write(store, data)
data = defaultdict(lambda: defaultdict(list))
write(store, data)
assert gain in ["high", "low"]
all_slopes = [[] for i in range(1024)]
#fig, a = plt.subplots(1)
for event in tqdm(event_generator):
event = calib(event)
calibrated = event.data[pixel][gain]
zero_crossings = np.where(np.diff(np.signbit(calibrated)))[0]
slope = calibrated[zero_crossings + 1] - calibrated[zero_crossings]
zero_crossing_cells = dr.sample2cell(zero_crossings,
stop_cell=event.header.stop_cells[pixel][gain],
total_cells=1024)
for crossing_number, cell_id in enumerate(zero_crossing_cells):
all_slopes[cell_id].append(np.abs(slope[crossing_number]))
slope_mean = np.zeros(1024)
slope_std = np.zeros(1024)
for cell_id, slopes in enumerate(all_slopes):
slopes = np.array(slopes)
slope_mean[cell_id] = slopes.mean()
slope_std[cell_id] = slopes.std() / np.sqrt(len(slopes))
print(slope_std)
df = pd.DataFrame({
"cell_width_mean": slope_mean / slope_mean.mean(),
arrival_time_no_calib = np.zeros(NN, dtype='f4')
trapz = np.zeros(NN, dtype='f4')
simps = np.zeros(NN, dtype='f4')
for i, event in enumerate(progress_bar(run, leave=True)):
raw_data = event.data[ch][gain]
stop_cell = event.header.stop_cells[ch][gain]
calibrated = raw_data - offset[stop_cell:stop_cell + run.roi]
t = cell_width[stop_cell:stop_cell + run.roi].cumsum()
max_pos[i] = np.argmax(calibrated)
s = slice(max_pos[i] - half_integration_window,
max_pos[i] + half_integration_window + 1)
samples = np.arange(s.start, s.stop)
cells = dr.sample2cell(samples, stop_cell, total_cells=1024)
DLE = partial(digital_leading_edge_discriminator,
data=calibrated,
threshold=1000)
arrival_time[i] = DLE(time=t)
arrival_time_no_calib[i] = DLE(time=np.arange(len(calibrated)))
integral[i] = calibrated[s].sum()
integral_weighted[i] = (calibrated[s] * midpoint_width[cells]).sum()
trapz[i] = np.trapz(calibrated[s], t[s])
simps[i] = scipy.integrate.simps(calibrated[s], t[s])
df = pd.DataFrame({
"integral": integral,
"integral_weighted": integral_weighted,
"max_pos": max_pos,
def main():
args = docopt(__doc__)
args["--max_events"] = None if args["--max_events"] is None else int(
args["--max_events"])
args["--skip_begin"] = int(args["--skip_begin"])
args["--skip_end"] = int(args["--skip_end"])
args["--do_channel8"] = bool(args["--do_channel8"])
print(args['<outputfile>'])
if os.path.isfile(args['<outputfile>']):
answer = input(
'Outputfile {} exists. Do you want to overwrite? y/[n] '.format(
args['<outputfile>']))
if not answer.lower().startswith('y'):
sys.exit()
store = pd.HDFStore(args['<outputfile>'], 'w')
adc = {}
delta_t = {}
for pixel in range(8 if args["--do_channel8"] else 7):
for gain in ["high", "low"]:
adc[pixel, gain] = [array('H') for i in range(4096)]
delta_t[pixel, gain] = [array('f') for i in range(4096)]
if not args["--do_channel8"]:
# We need to put nans for channel 8 into the output file, since the
# rest of the system expects this data to be there ... even if it its nan.
result = pd.DataFrame(np.full((4096, 4), np.nan),
columns=['a', 'b', 'c', 'chisq_ndf'])
result['pixel'] = 7
result['channel'] = "high"
result['cell'] = np.arange(4096)
store.append('data', result, min_itemsize={'channel': 4})
result['channel'] = "low"
store.append('data', result, min_itemsize={'channel': 4})
print("reading raw file(s) into memory:")
for eg in [
dr.EventGenerator(filename, max_events=args["--max_events"])
for filename in args["<inputfiles>"]
]:
sample_ids = np.arange(eg.roi)
for event in tqdm(
iterable=eg,
desc=os.path.basename(eg.path),
leave=True,
unit=' events',
):
for pixel, gain in sorted(adc.keys()):
sc = event.header.stop_cells[pixel][gain]
skip_slice = slice(args["--skip_begin"], -args["--skip_end"])
cell_ids = dr.sample2cell(sample_ids, sc)[skip_slice]
data = event.data[pixel][gain][skip_slice]
delta_ts = event.time_since_last_readout[pixel][gain][
skip_slice]
for i, cid in enumerate(cell_ids):
delta_t[pixel, gain][cid].append(delta_ts[i])
adc[pixel, gain][cid].append(data[i])
for key in sorted(adc.keys()):
# adc and delta_t have the same keys
for i in tqdm(range(4096), desc=str(key)):
adc[key][i] = np.array(adc[key][i], dtype=adc[key][i].typecode)
delta_t[key][i] = np.array(delta_t[key][i],
dtype=delta_t[key][i].typecode)
adc[key][i] = adc[key][i][~np.isnan(delta_t[key][i])]
delta_t[key][i] = delta_t[key][i][~np.isnan(delta_t[key][i])]
print("fitting")
pool = Parallel(max(psutil.cpu_count() - 1, 1))
for key in tqdm(iterable=sorted(adc.keys()), leave=True):
result = pd.DataFrame(pool(
delayed(fit)(adc[key][i], delta_t[key][i], i)
for i in range(4096)),
columns=['a', 'b', 'c', 'chisq_ndf'])
pixel, channel = key
result['pixel'] = pixel
result['channel'] = channel
result['cell'] = np.arange(4096)
store.append('data', result, min_itemsize={'channel': 4})
