def get_set_spectrum(set_abs_path, borders=None, bins=30):
"""Calculate energy spectrum for set."""
points = natsorted(glob.glob(path.join(set_abs_path, "p*.df")))
out = {}
for point in points:
_, meta, data = dfparser.parse_from_file(point)
parsed_data = dfparser.Point()
parsed_data.ParseFromString(data)
del data
amps = []
times = []
for channel in parsed_data.channels:
for block in channel.blocks:
amps.append(np.array(block.events.amplitudes, np.int16))
times.append(np.array(block.events.times, np.uint64))
amps = np.hstack(amps)
times = np.hstack(times)
hist, bins = np.histogram(amps, bins, range=borders, density=True)
hist_unnorm, _ = np.histogram(amps, bins, range=borders)
out[path.relpath(point, set_abs_path)] = {
"meta": meta,
"hist": hist,
"hist_unnorm": hist_unnorm,
"bins": bins
}
return out
def get_amps(filepath: str) -> np.ndarray:
"""Extract amplitudes from processed file."""
_, _, data = dfparser.parse_from_file(filepath)
p_high = dfparser.Point()
p_high.ParseFromString(data)
amps = np.hstack(
[list(block.events.amplitudes) for block in p_high.channels[0].blocks])
return amps
def _main():
# Parse arguments from command line
args = __parse_args()
points = natsorted(glob.glob(path.join(args.data_root, args.set, "p*.df")))
amps = {}
for p in points:
# Read dataforge point
_, meta, data = dfparser.parse_from_file(p)
# Parse Binary data
point = dfparser.Point()
point.ParseFromString(data)
hv = int(meta['external_meta']['HV1_value'])
time = meta['params']['events_num'] * meta['params']['b_size'] / \
meta['params']['sample_freq']
if hv not in amps:
amps[hv] = {
'time': 0,
'amps': []
}
amps[hv]['time'] += time
for idx, channel in enumerate(point.channels):
for block in channel.blocks:
amps[hv]['amps'].append(np.array(
block.events.amplitudes, np.int16))
for hv in amps:
amps[hv]['amps'] = np.hstack(amps[hv]['amps'])
amps[hv]['count_rate'] = len(amps[hv]['amps'][
amps[hv]['amps'] >= args.ampl_threshold]) / amps[hv]['time']
hvs = sorted(list(amps.keys()))
crs = [amps[hv]['count_rate'] for hv in hvs]
_, axes = plt.subplots()
if args.x_scale:
axes.set_xscale(args.x_scale)
if args.y_scale:
axes.set_yscale(args.y_scale)
axes.set_title(args.set)
axes.set_xlabel("High voltage, V")
axes.set_ylabel("Count rate, Ev/s")
axes.plot(hvs, crs, label='Set points count rate')
axes.legend()
plt.show()
def _lan_amps(data):
point = dfparser.Point()
point.ParseFromString(data)
amps = []
for idx, channel in enumerate(point.channels):
for block in channel.blocks:
amps.append(np.array(block.events.amplitudes, np.int16))
return np.hstack(amps)
def _main():
# Parse arguments from command line
args = __parse_args()
# Read dataforge point
_, meta, data = dfparser.parse_from_file(args.input)
# Parse Binary data
point = dfparser.Point()
point.ParseFromString(data)
# Extract amlitudes from each block
amps = {}
for idx, channel in enumerate(point.channels):
for block in channel.blocks:
if idx not in amps:
amps[idx] = []
amps[idx].append(np.array(block.events.amplitudes, np.int16))
for idx in amps:
amps[idx] = np.hstack(amps[idx])
if not args.split_channels:
plots = {
"all-channels": np.hstack(amps.values())
}
else:
plots = amps
_, axes = plt.subplots()
axes.set_title(args.input)
axes.set_xlabel("Channels, ch")
axes.set_ylabel("Counts")
for idx, plot in enumerate(plots):
# Calculate histogram
hist, bins = np.histogram(
plots[plot], bins=args.bins, range=(
args.ampl_threshold, args.ampl_max))
# Calculate bins centers
bins_centers = (bins[:-1] + bins[1:]) / 2
# Drawing graph
label = idx
if "channels" in meta:
if str(idx) in meta["channels"]:
label = meta["channels"][str(idx)]
axes.step(bins_centers, hist, where='mid', label=label)
axes.legend()
plt.show()
def df_frames_to_events(meta, data, extract_func, frame_l=15, frame_r=25,
correct_time=False):
"""Convert frames to events in dataforge points."""
threshold = meta['process_params']['threshold']
sample_freq = meta['params']['sample_freq']
point = dfparser.Point()
point.ParseFromString(data)
for channel in point.channels:
for block in channel.blocks:
events = block.events
for _ in range(len(block.frames)):
frame = block.frames.pop(0)
if correct_time:
frame.time = (frame.time // np.uint64(10)
).astype(np.uint64)
ev_data = np.frombuffer(frame.data, np.int16)
params, singles_raw = extract_func(ev_data, frame.time,
threshold, sample_freq)
singles = np.where(singles_raw == True)[0]
events.times.extend(np.round(params[singles * 2 + 1])
.astype(np.uint64))
events.amplitudes.extend(np.round(params[singles * 2])
.astype(np.uint64))
doubles = np.where(singles_raw == False)[0]
events_bins = np.round((params[doubles * 2 + 1] -
frame.time) * sample_freq / 1e+9)
frames_block = extract_frames(ev_data, events_bins,
frame_l, frame_r)
for idx, frame in enumerate(frames_block):
time = int(np.round(params[doubles[idx] * 2 + 1]) +
frame_l / sample_freq * 1e+9)
frame_ser = frame.astype(np.int16).tobytes()
block.frames.add(time=time, data=frame_ser)
meta['process_params']['extracted'] = True
meta['process_params']['extracted_frame_l'] = frame_l
meta['process_params']['extracted_frame_r'] = frame_r
return meta, point.SerializeToString()
def __main():
dist_file = path.abspath(
path.join(path.dirname(__file__), "../../signal_utils/data/dist.dat"))
meta, data, answer = generate_df(time=TIME_SEC,
dist_file=dist_file,
dist_time_func=gen_doubles,
freq=10e3)
ev_times = answer[1::2]
ev_time_diffs = ev_times[1:] - ev_times[:-1]
_, dist_hist_ax = plt.subplots()
dist_hist_ax.set_title("Histogram of distances between generated events")
dist_hist_ax.set_xlabel("Distance, ns")
dist_hist_ax.set_ylabel("Events number")
hist, bins = np.histogram(ev_time_diffs,
bins=HIST_DIFF_BINS_N,
range=HIST_DIFF_RANGE)
dist_hist_ax.step((bins[1:] + bins[:-1]) / 2, hist)
_, data_ext = df_frames_to_events(meta, data, extract_amps_approx2)
point = dfparser.Point()
point.ParseFromString(data_ext)
evs_recovered = []
for channel in point.channels:
for block in channel.blocks:
evs_recovered.append(np.array(block.events.amplitudes))
evs_recovered = np.hstack(evs_recovered)
_, ampl_hist_ax = plt.subplots()
ampl_hist_ax.set_title("Events amplutides histogram")
ampl_hist_ax.set_xlabel("Amplitude, ch")
ampl_hist_ax.set_ylabel("Events number")
hist, bins = np.histogram(answer[0::2],
bins=HIST_EVS_BINS_N,
range=HIST_EVS_RANGE)
ampl_hist_ax.step((bins[1:] + bins[:-1]) / 2,
hist,
where='mid',
label="Generated events")
hist, bins = np.histogram(evs_recovered,
bins=HIST_EVS_BINS_N,
range=HIST_EVS_RANGE)
ampl_hist_ax.step((bins[1:] + bins[:-1]) / 2,
hist,
where='mid',
label="Reconstructed events")
ampl_hist_ax.legend()
def df_events_to_np(_, data):
"""Конвертация массива точек из формата df в numpy array."""
point = dfparser.Point()
point.ParseFromString(data)
amps = []
times = []
channel = point.channels[0]
for block in channel.blocks:
amps.append(block.events.amplitudes)
times.append((np.array(block.events.times) + block.time))
amps = np.hstack(amps)
times = np.hstack(times)
return amps, times
def __extract_amps(filename):
_, meta, data = dfparser.parse_from_file(filename)
# Parse Binary data
point = dfparser.Point()
point.ParseFromString(data)
# Extract amlitudes from each block
amps = {}
for idx, channel in enumerate(point.channels):
for block in channel.blocks:
if idx not in amps:
amps[idx] = []
amps[idx].append(np.array(block.events.amplitudes, np.int16))
for idx in amps:
amps[idx] = np.hstack(amps[idx])
return np.hstack(amps.values())
def _main():
args = _parse_args()
_, _, data = dfparser.parse_from_file(args.input)
point = dfparser.Point()
point.ParseFromString(data)
del data
datas = []
for channel in point.channels:
amps = channel.blocks[0].events.amplitudes
times = channel.blocks[0].events.times
data = np.zeros((len(amps), 3))
data[:, 0] = list(times)
data[:, 1] = list(amps)
datas.append(data)
datas = np.vstack(datas)
datas = datas[datas[:, 0].argsort()]
datas[:, 2][1:] = datas[:, 0][1:] - datas[:, 0][:-1]
returned = datas[datas[:, 2] < args.threshold, :]
delta_max = args.delta_max
if not delta_max:
delta_max = args.threshold
returned = returned[np.logical_and(returned[:, 1] >= args.amp_min,
returned[:, 1] <= args.amp_max)]
returned = returned[np.logical_and(returned[:, 2] >= args.delta_min,
returned[:, 2] <= args.delta_max)]
plot = sns.jointplot(
returned[:, 2],
returned[:, 1],
kind="hex",
stat_func=None,
xlim=(args.delta_min, delta_max),
ylim=(args.amp_min, args.amp_max),
joint_kws={'gridsize': (args.bins_delta, args.bins_amp)})
plot.set_axis_labels("Time delta, ns", "Amplitude, ch")
plt.show()
def _main():
# Parse arguments from command line
args = __parse_args()
# Read dataforge point
_, _, data = dfparser.parse_from_file(args.input)
# Parse Binary data
point = dfparser.Point()
point.ParseFromString(data)
# Extract event times from each block
times = []
for channel in point.channels:
for block in channel.blocks:
times.append(np.array(block.events.times, np.uint64))
# Combine times into one array
times = np.hstack(times)
# Calculate time differences
diffs = times[1:] - times[:-1]
# Calculate histogram
hist, bins = np.histogram(diffs,
bins=args.bins,
range=(args.ampl_threshold, args.ampl_max))
# Calculate bins centers
bins_centers = (bins[:-1] + bins[1:]) / 2
# Drawing graph
_, axes = plt.subplots()
axes.set_title(args.input)
axes.set_xlabel("Time, ns")
axes.set_ylabel("Counts")
axes.step(bins_centers, hist, where='mid')
plt.show()
def _main():
args = _parse_args()
files = natsorted(listdir(args.input))
point = dfparser.Point()
channels = {}
meta = {"compression": "zlib", "channels": {}}
for filename in files:
if not filename.endswith('.mat'):
continue
data = loadmat(path.join(args.input, filename))
stats_raw = data["statistics"]
stats = {
str(stats_raw[0, i][0]): stats_raw[1, i].tolist()
for i in range(stats_raw.shape[1])
}
for _ in range(2):
stats = {
val: stats[val][0]
if stats[val] and isinstance(stats[val], list) else stats[val]
for val in stats
}
channel = stats['Board identifier/Spectrum ID']
if channel in args.exclude:
continue
if channel not in channels:
channels[channel] = point.channels.add()
channels[channel].blocks.add()
meta["channels"][len(meta["channels"])] = channel
amps = channels[channel].blocks[0].events.amplitudes
times = channels[channel].blocks[0].events.times
for amp, time in zip([d[0] for d in data['energies']],
[d[0] * BIN_SIZE_NS for d in data['timestamps']]):
amps.append(int(amp))
times.append(int(time))
msg = dfparser.create_message(meta, point.SerializeToString())
out_file = path.join(args.input, 'processed')
if args.output:
out_file = args.output
with open("%s.df" % out_file, 'wb') as out:
out.write(msg)
if args.ascii:
datas = []
for idx, channel in enumerate(point.channels):
amps = channel.blocks[0].events.amplitudes
times = channel.blocks[0].events.times
data = np.zeros((len(amps), 3))
data[:, 0] = list(times)
data[:, 1] = list(amps)
data[:, 2] = idx
datas.append(data)
datas = np.vstack(datas)
datas = datas[datas[:, 0].argsort()]
np.savetxt("%s.txt" % out_file, datas, fmt='%.4e')
if __name__ == "__main__":
seaborn.set_context("poster")
args = _parse_args()
if not args.input:
if args.root == 'gen':
meta, data, _ = generate_df(
area_l=AREA_L,
area_r=AREA_R,
time=float(args.wildcard),
dist_file=abspath(
join(dirname(__file__),
'../../signal_utils/data/dist.dat')))
point = dfparser.Point()
point.ParseFromString(data)
chi2s = _extr_chi2(point)
else:
files = glob(join(args.root, args.wildcard))
chi2s = np.array([])
for p in tqdm(files):
point = dfparser.Point()
_, meta, data = dfparser.parse_from_file(p)
point.ParseFromString(data)
chi2s = np.append(chi2s, _extr_chi2(point))
np.save(args.output, chi2s)
else:
chi2s = np.load(args.input)
def rsb_to_df(ext_meta: dict, rsb_file, threshold: int=500,
area_l: int=50, area_r: int=100) -> (dict, bytearray, int):
"""Конвертировние данных формата rsb в формат df.
@meta - метаданные сообщения с точками
@rsb_file - файл с платы Руднева-Шиляева
@threshold - порог амплитуды события (параметр zero-suppression)
@area_l - область около события, которая будет сохранена (параметр
zero-suppression)
@area_r - область около события, которая будет сохранена (параметр
zero-suppression)
@return - (meta, data, data_type)
"""
sec_coef = 1e+9
rsb_ds = dfparser.RshPackage(rsb_file)
meta = {}
meta["external_meta"] = ext_meta
meta["params"] = rsb_ds.params
meta["process_params"] = {
"threshold": threshold,
"area_l": area_l,
"area_r": area_r
}
begin_time = parse(rsb_ds.params["start_time"]).timestamp() * sec_coef
end_time = parse(rsb_ds.params["end_time"]).timestamp() * sec_coef
bin_time = (rsb_ds.params["sample_freq"]**-1) * sec_coef
b_size = rsb_ds.params["b_size"]
if rsb_ds.params["events_num"] == -1:
meta["recalc_events_num"] = True
rsb_ds.params["events_num"] = np.iinfo(int).max
for i in range(np.iinfo(int).max):
try:
rsb_ds.get_event(i)
except Exception as exception:
rsb_ds.params["events_num"] = i
break
events_num = rsb_ds.params["events_num"]
ch_num = rsb_ds.params["channel_number"]
use_time_corr = False
if events_num > 0:
event = rsb_ds.get_event(0)
if "ns_since_epoch" not in event:
use_time_corr = True
times = list(np.linspace(begin_time, end_time -
int(bin_time * b_size),
events_num))
meta["correcting_time"] = "linear"
point = dfparser.Point()
channels = [point.channels.add(id=channel) for channel in range(ch_num)]
for i in range(events_num):
event_data = rsb_ds.get_event(i)
if use_time_corr:
time = times[i]
else:
time = event_data["ns_since_epoch"]
for channel in range(ch_num):
block = channels[channel].blocks.add(time=int(time), )
ch_data = event_data["data"][channel::ch_num]
for frame in apply_zsupression(ch_data, threshold, area_l, area_r):
frame = np.clip(frame, 0, ch_data.shape[0] - 1)
event = block.frames.add()
event.time = int(frame[0] * bin_time)
event.data = ch_data[frame[0]:frame[1]].astype(np.int16) \
.tobytes()
meta["bin_offset"] = 0
meta["bin_size"] = point.ByteSize()
data = point.SerializeToString()
return meta, data
def main():
"""Execute main function."""
df_madc_data_root = "/home/chernov/data_on_server_madc"
df_data_root = "/home/chernov/data_processed"
set_path = "2017_05/Fill_2/set_8"
threshold_madc = 450
threshold_madc_h = 3100
threshold = threshold_madc * 2.03
threshold_h = threshold_madc_h * 2.03
points = sorted(glob(path.join(df_data_root, set_path, "p**")))
points_madc = sorted(glob(path.join(df_madc_data_root, set_path, "p**")))
counts_madc = {}
for point in points_madc:
_, meta, data = dfparser.parse_from_file(point)
hv_val = int(meta['external_meta']['HV1_value'])
if hv_val not in counts_madc:
counts_madc[hv_val] = []
amps = parse_madc_binary(data)
filt = np.logical_and(amps > threshold_madc, amps < threshold_madc_h)
counts_madc[hv_val].append(amps[filt].size)
counts = {}
for point in points:
_, meta, data = dfparser.parse_from_file(point)
hv_val = int(meta['external_meta']['HV1_value'])
time = meta['params']['b_size'] * meta['params']['events_num'] / \
meta['params']['sample_freq']
point_ds = dfparser.Point()
point_ds.ParseFromString(data)
amps = np.hstack([
list(block.events.amplitudes)
for block in point_ds.channels[0].blocks
])
if hv_val not in counts:
counts[hv_val] = []
filt = np.logical_and(amps > threshold, amps < threshold_h)
counts[hv_val].append(float(amps[filt].size) / time * 30.0)
counts = {key: np.mean(counts[key]) for key in counts}
counts_madc = {key: np.mean(counts_madc[key]) for key in counts_madc}
counts_x = list(counts.keys())
counts_y = [counts[key] for key in counts.keys()]
counts_madc_x = list(counts_madc.keys())
counts_madc_y = [counts_madc[key] for key in counts_madc.keys()]
fig, axes = plt.subplots()
fig.canvas.set_window_title("energy_spectrum_compare")
fig.suptitle("CAMAC MADC vs. Lan10-12PCI energy spectrums \n"
"Set - %s" % (set_path))
axes.plot(counts_x, counts_y, 'ro', label='Lan10-12PCI')
axes.plot(counts_madc_x, counts_madc_y, 'bo', label='MADC')
axes.set_xlabel("Voltage, V")
axes.set_ylabel("Efficient counts")
axes.legend()
print(counts_x)
print(counts_madc_x)
# assert counts_x == counts_madc_x
fig, axes = plt.subplots()
fig.canvas.set_window_title("energy_spectrum_compare_ratio")
fig.suptitle("CAMAC MADC vs. Lan10-12PCI energy spectrums ratio \n"
"Set - %s" % (set_path))
axes.plot(counts_x,
1 - np.array(counts_y) / np.array(counts_madc_y),
'ro',
label='')
axes.set_xlabel("Voltage, V")
axes.set_ylabel("MADS vs Lan10_12PCI ratio")
def main():
"""Compare Lan10-12PCI points energy spectrum for several points."""
seaborn.set_context("poster")
data = np.genfromtxt("/home/chernov/Downloads/set_43_detector.out")
x_data = data[:, 0]
y_points = data[:, 1:].transpose()
y_points = (y_points.T / np.max(y_points, axis=1)).T
df_data_root = "/home/chernov/data_processed"
points_path = [
"2017_05/Fill_3/set_43/p0(30s)(HV1=16000).df",
"2017_05/Fill_3/set_43/p36(30s)(HV1=17000).df",
"2017_05/Fill_3/set_43/p80(30s)(HV1=15000).df",
"2017_05/Fill_3/set_43/p102(30s)(HV1=14000).df"
]
bins = 500
range_ = (0, 8000)
for idx, point_rel in enumerate(points_path):
_, _, data = dfparser.parse_from_file(
path.join(df_data_root, point_rel))
point = dfparser.Point()
point.ParseFromString(data)
amps = np.hstack([
list(block.events.amplitudes) for block in point.channels[0].blocks
])
hist, x_point = np.histogram(amps, bins=bins, range=range_)
hist = hist / np.max(hist[np.where(x_point > 3000)[0][0]:])
func = interp1d(x_point[1:] + x_point[:-1],
hist,
bounds_error=False,
fill_value=0)
func_mod = lambda x, a, b, c: c * func(a * x + b)
x_peak = np.where(np.logical_and(x_data > 1000, x_data < 1600))
popt, _ = curve_fit(func_mod,
x_data[x_peak],
y_points[idx][x_peak],
p0=[3.68, 700, 1])
fig, axes = plt.subplots()
fig.canvas.set_window_title(point_rel)
fig.suptitle("CAMAC MADC vs. Lan10-12PCI spectrums")
axes.set_title("File - %s. \nOptimized parameters: a=%s, b=%s, c=%s" %
(point_rel, *np.round(popt, 2)))
axes.set_xlabel("Bins, ch")
axes.set_xlim(0, 2000)
# axes.set_yscale("log", nonposx='clip')
x_interp = np.linspace(0, 2000, 500)
axes.plot(x_interp, func_mod(x_interp, *popt), label="Lan10-12PCI")
axes.plot(x_data, y_points[idx], label="CAMAC MADC")
axes.legend()
