def get_set_spectrum(set_abs_path, borders=None, bins=30): """Calculate energy spectrum for set.""" points = natsorted(glob.glob(path.join(set_abs_path, "p*.df"))) out = {} for point in points: _, meta, data = dfparser.parse_from_file(point) parsed_data = dfparser.Point() parsed_data.ParseFromString(data) del data amps = [] times = [] for channel in parsed_data.channels: for block in channel.blocks: amps.append(np.array(block.events.amplitudes, np.int16)) times.append(np.array(block.events.times, np.uint64)) amps = np.hstack(amps) times = np.hstack(times) hist, bins = np.histogram(amps, bins, range=borders, density=True) hist_unnorm, _ = np.histogram(amps, bins, range=borders) out[path.relpath(point, set_abs_path)] = { "meta": meta, "hist": hist, "hist_unnorm": hist_unnorm, "bins": bins } return out
def get_amps(filepath: str) -> np.ndarray: """Extract amplitudes from processed file.""" _, _, data = dfparser.parse_from_file(filepath) p_high = dfparser.Point() p_high.ParseFromString(data) amps = np.hstack( [list(block.events.amplitudes) for block in p_high.channels[0].blocks]) return amps
def _main(): # Parse arguments from command line args = __parse_args() points = natsorted(glob.glob(path.join(args.data_root, args.set, "p*.df"))) amps = {} for p in points: # Read dataforge point _, meta, data = dfparser.parse_from_file(p) # Parse Binary data point = dfparser.Point() point.ParseFromString(data) hv = int(meta['external_meta']['HV1_value']) time = meta['params']['events_num'] * meta['params']['b_size'] / \ meta['params']['sample_freq'] if hv not in amps: amps[hv] = { 'time': 0, 'amps': [] } amps[hv]['time'] += time for idx, channel in enumerate(point.channels): for block in channel.blocks: amps[hv]['amps'].append(np.array( block.events.amplitudes, np.int16)) for hv in amps: amps[hv]['amps'] = np.hstack(amps[hv]['amps']) amps[hv]['count_rate'] = len(amps[hv]['amps'][ amps[hv]['amps'] >= args.ampl_threshold]) / amps[hv]['time'] hvs = sorted(list(amps.keys())) crs = [amps[hv]['count_rate'] for hv in hvs] _, axes = plt.subplots() if args.x_scale: axes.set_xscale(args.x_scale) if args.y_scale: axes.set_yscale(args.y_scale) axes.set_title(args.set) axes.set_xlabel("High voltage, V") axes.set_ylabel("Count rate, Ev/s") axes.plot(hvs, crs, label='Set points count rate') axes.legend() plt.show()
def _lan_amps(data): point = dfparser.Point() point.ParseFromString(data) amps = [] for idx, channel in enumerate(point.channels): for block in channel.blocks: amps.append(np.array(block.events.amplitudes, np.int16)) return np.hstack(amps)
def _main(): # Parse arguments from command line args = __parse_args() # Read dataforge point _, meta, data = dfparser.parse_from_file(args.input) # Parse Binary data point = dfparser.Point() point.ParseFromString(data) # Extract amlitudes from each block amps = {} for idx, channel in enumerate(point.channels): for block in channel.blocks: if idx not in amps: amps[idx] = [] amps[idx].append(np.array(block.events.amplitudes, np.int16)) for idx in amps: amps[idx] = np.hstack(amps[idx]) if not args.split_channels: plots = { "all-channels": np.hstack(amps.values()) } else: plots = amps _, axes = plt.subplots() axes.set_title(args.input) axes.set_xlabel("Channels, ch") axes.set_ylabel("Counts") for idx, plot in enumerate(plots): # Calculate histogram hist, bins = np.histogram( plots[plot], bins=args.bins, range=( args.ampl_threshold, args.ampl_max)) # Calculate bins centers bins_centers = (bins[:-1] + bins[1:]) / 2 # Drawing graph label = idx if "channels" in meta: if str(idx) in meta["channels"]: label = meta["channels"][str(idx)] axes.step(bins_centers, hist, where='mid', label=label) axes.legend() plt.show()
def df_frames_to_events(meta, data, extract_func, frame_l=15, frame_r=25, correct_time=False): """Convert frames to events in dataforge points.""" threshold = meta['process_params']['threshold'] sample_freq = meta['params']['sample_freq'] point = dfparser.Point() point.ParseFromString(data) for channel in point.channels: for block in channel.blocks: events = block.events for _ in range(len(block.frames)): frame = block.frames.pop(0) if correct_time: frame.time = (frame.time // np.uint64(10) ).astype(np.uint64) ev_data = np.frombuffer(frame.data, np.int16) params, singles_raw = extract_func(ev_data, frame.time, threshold, sample_freq) singles = np.where(singles_raw == True)[0] events.times.extend(np.round(params[singles * 2 + 1]) .astype(np.uint64)) events.amplitudes.extend(np.round(params[singles * 2]) .astype(np.uint64)) doubles = np.where(singles_raw == False)[0] events_bins = np.round((params[doubles * 2 + 1] - frame.time) * sample_freq / 1e+9) frames_block = extract_frames(ev_data, events_bins, frame_l, frame_r) for idx, frame in enumerate(frames_block): time = int(np.round(params[doubles[idx] * 2 + 1]) + frame_l / sample_freq * 1e+9) frame_ser = frame.astype(np.int16).tobytes() block.frames.add(time=time, data=frame_ser) meta['process_params']['extracted'] = True meta['process_params']['extracted_frame_l'] = frame_l meta['process_params']['extracted_frame_r'] = frame_r return meta, point.SerializeToString()
def __main(): dist_file = path.abspath( path.join(path.dirname(__file__), "../../signal_utils/data/dist.dat")) meta, data, answer = generate_df(time=TIME_SEC, dist_file=dist_file, dist_time_func=gen_doubles, freq=10e3) ev_times = answer[1::2] ev_time_diffs = ev_times[1:] - ev_times[:-1] _, dist_hist_ax = plt.subplots() dist_hist_ax.set_title("Histogram of distances between generated events") dist_hist_ax.set_xlabel("Distance, ns") dist_hist_ax.set_ylabel("Events number") hist, bins = np.histogram(ev_time_diffs, bins=HIST_DIFF_BINS_N, range=HIST_DIFF_RANGE) dist_hist_ax.step((bins[1:] + bins[:-1]) / 2, hist) _, data_ext = df_frames_to_events(meta, data, extract_amps_approx2) point = dfparser.Point() point.ParseFromString(data_ext) evs_recovered = [] for channel in point.channels: for block in channel.blocks: evs_recovered.append(np.array(block.events.amplitudes)) evs_recovered = np.hstack(evs_recovered) _, ampl_hist_ax = plt.subplots() ampl_hist_ax.set_title("Events amplutides histogram") ampl_hist_ax.set_xlabel("Amplitude, ch") ampl_hist_ax.set_ylabel("Events number") hist, bins = np.histogram(answer[0::2], bins=HIST_EVS_BINS_N, range=HIST_EVS_RANGE) ampl_hist_ax.step((bins[1:] + bins[:-1]) / 2, hist, where='mid', label="Generated events") hist, bins = np.histogram(evs_recovered, bins=HIST_EVS_BINS_N, range=HIST_EVS_RANGE) ampl_hist_ax.step((bins[1:] + bins[:-1]) / 2, hist, where='mid', label="Reconstructed events") ampl_hist_ax.legend()
def df_events_to_np(_, data): """Конвертация массива точек из формата df в numpy array.""" point = dfparser.Point() point.ParseFromString(data) amps = [] times = [] channel = point.channels[0] for block in channel.blocks: amps.append(block.events.amplitudes) times.append((np.array(block.events.times) + block.time)) amps = np.hstack(amps) times = np.hstack(times) return amps, times
def __extract_amps(filename): _, meta, data = dfparser.parse_from_file(filename) # Parse Binary data point = dfparser.Point() point.ParseFromString(data) # Extract amlitudes from each block amps = {} for idx, channel in enumerate(point.channels): for block in channel.blocks: if idx not in amps: amps[idx] = [] amps[idx].append(np.array(block.events.amplitudes, np.int16)) for idx in amps: amps[idx] = np.hstack(amps[idx]) return np.hstack(amps.values())
def _main(): args = _parse_args() _, _, data = dfparser.parse_from_file(args.input) point = dfparser.Point() point.ParseFromString(data) del data datas = [] for channel in point.channels: amps = channel.blocks[0].events.amplitudes times = channel.blocks[0].events.times data = np.zeros((len(amps), 3)) data[:, 0] = list(times) data[:, 1] = list(amps) datas.append(data) datas = np.vstack(datas) datas = datas[datas[:, 0].argsort()] datas[:, 2][1:] = datas[:, 0][1:] - datas[:, 0][:-1] returned = datas[datas[:, 2] < args.threshold, :] delta_max = args.delta_max if not delta_max: delta_max = args.threshold returned = returned[np.logical_and(returned[:, 1] >= args.amp_min, returned[:, 1] <= args.amp_max)] returned = returned[np.logical_and(returned[:, 2] >= args.delta_min, returned[:, 2] <= args.delta_max)] plot = sns.jointplot( returned[:, 2], returned[:, 1], kind="hex", stat_func=None, xlim=(args.delta_min, delta_max), ylim=(args.amp_min, args.amp_max), joint_kws={'gridsize': (args.bins_delta, args.bins_amp)}) plot.set_axis_labels("Time delta, ns", "Amplitude, ch") plt.show()
def _main(): # Parse arguments from command line args = __parse_args() # Read dataforge point _, _, data = dfparser.parse_from_file(args.input) # Parse Binary data point = dfparser.Point() point.ParseFromString(data) # Extract event times from each block times = [] for channel in point.channels: for block in channel.blocks: times.append(np.array(block.events.times, np.uint64)) # Combine times into one array times = np.hstack(times) # Calculate time differences diffs = times[1:] - times[:-1] # Calculate histogram hist, bins = np.histogram(diffs, bins=args.bins, range=(args.ampl_threshold, args.ampl_max)) # Calculate bins centers bins_centers = (bins[:-1] + bins[1:]) / 2 # Drawing graph _, axes = plt.subplots() axes.set_title(args.input) axes.set_xlabel("Time, ns") axes.set_ylabel("Counts") axes.step(bins_centers, hist, where='mid') plt.show()
def _main(): args = _parse_args() files = natsorted(listdir(args.input)) point = dfparser.Point() channels = {} meta = {"compression": "zlib", "channels": {}} for filename in files: if not filename.endswith('.mat'): continue data = loadmat(path.join(args.input, filename)) stats_raw = data["statistics"] stats = { str(stats_raw[0, i][0]): stats_raw[1, i].tolist() for i in range(stats_raw.shape[1]) } for _ in range(2): stats = { val: stats[val][0] if stats[val] and isinstance(stats[val], list) else stats[val] for val in stats } channel = stats['Board identifier/Spectrum ID'] if channel in args.exclude: continue if channel not in channels: channels[channel] = point.channels.add() channels[channel].blocks.add() meta["channels"][len(meta["channels"])] = channel amps = channels[channel].blocks[0].events.amplitudes times = channels[channel].blocks[0].events.times for amp, time in zip([d[0] for d in data['energies']], [d[0] * BIN_SIZE_NS for d in data['timestamps']]): amps.append(int(amp)) times.append(int(time)) msg = dfparser.create_message(meta, point.SerializeToString()) out_file = path.join(args.input, 'processed') if args.output: out_file = args.output with open("%s.df" % out_file, 'wb') as out: out.write(msg) if args.ascii: datas = [] for idx, channel in enumerate(point.channels): amps = channel.blocks[0].events.amplitudes times = channel.blocks[0].events.times data = np.zeros((len(amps), 3)) data[:, 0] = list(times) data[:, 1] = list(amps) data[:, 2] = idx datas.append(data) datas = np.vstack(datas) datas = datas[datas[:, 0].argsort()] np.savetxt("%s.txt" % out_file, datas, fmt='%.4e')
if __name__ == "__main__": seaborn.set_context("poster") args = _parse_args() if not args.input: if args.root == 'gen': meta, data, _ = generate_df( area_l=AREA_L, area_r=AREA_R, time=float(args.wildcard), dist_file=abspath( join(dirname(__file__), '../../signal_utils/data/dist.dat'))) point = dfparser.Point() point.ParseFromString(data) chi2s = _extr_chi2(point) else: files = glob(join(args.root, args.wildcard)) chi2s = np.array([]) for p in tqdm(files): point = dfparser.Point() _, meta, data = dfparser.parse_from_file(p) point.ParseFromString(data) chi2s = np.append(chi2s, _extr_chi2(point)) np.save(args.output, chi2s) else: chi2s = np.load(args.input)
def rsb_to_df(ext_meta: dict, rsb_file, threshold: int=500, area_l: int=50, area_r: int=100) -> (dict, bytearray, int): """Конвертировние данных формата rsb в формат df. @meta - метаданные сообщения с точками @rsb_file - файл с платы Руднева-Шиляева @threshold - порог амплитуды события (параметр zero-suppression) @area_l - область около события, которая будет сохранена (параметр zero-suppression) @area_r - область около события, которая будет сохранена (параметр zero-suppression) @return - (meta, data, data_type) """ sec_coef = 1e+9 rsb_ds = dfparser.RshPackage(rsb_file) meta = {} meta["external_meta"] = ext_meta meta["params"] = rsb_ds.params meta["process_params"] = { "threshold": threshold, "area_l": area_l, "area_r": area_r } begin_time = parse(rsb_ds.params["start_time"]).timestamp() * sec_coef end_time = parse(rsb_ds.params["end_time"]).timestamp() * sec_coef bin_time = (rsb_ds.params["sample_freq"]**-1) * sec_coef b_size = rsb_ds.params["b_size"] if rsb_ds.params["events_num"] == -1: meta["recalc_events_num"] = True rsb_ds.params["events_num"] = np.iinfo(int).max for i in range(np.iinfo(int).max): try: rsb_ds.get_event(i) except Exception as exception: rsb_ds.params["events_num"] = i break events_num = rsb_ds.params["events_num"] ch_num = rsb_ds.params["channel_number"] use_time_corr = False if events_num > 0: event = rsb_ds.get_event(0) if "ns_since_epoch" not in event: use_time_corr = True times = list(np.linspace(begin_time, end_time - int(bin_time * b_size), events_num)) meta["correcting_time"] = "linear" point = dfparser.Point() channels = [point.channels.add(id=channel) for channel in range(ch_num)] for i in range(events_num): event_data = rsb_ds.get_event(i) if use_time_corr: time = times[i] else: time = event_data["ns_since_epoch"] for channel in range(ch_num): block = channels[channel].blocks.add(time=int(time), ) ch_data = event_data["data"][channel::ch_num] for frame in apply_zsupression(ch_data, threshold, area_l, area_r): frame = np.clip(frame, 0, ch_data.shape[0] - 1) event = block.frames.add() event.time = int(frame[0] * bin_time) event.data = ch_data[frame[0]:frame[1]].astype(np.int16) \ .tobytes() meta["bin_offset"] = 0 meta["bin_size"] = point.ByteSize() data = point.SerializeToString() return meta, data
def main(): """Execute main function.""" df_madc_data_root = "/home/chernov/data_on_server_madc" df_data_root = "/home/chernov/data_processed" set_path = "2017_05/Fill_2/set_8" threshold_madc = 450 threshold_madc_h = 3100 threshold = threshold_madc * 2.03 threshold_h = threshold_madc_h * 2.03 points = sorted(glob(path.join(df_data_root, set_path, "p**"))) points_madc = sorted(glob(path.join(df_madc_data_root, set_path, "p**"))) counts_madc = {} for point in points_madc: _, meta, data = dfparser.parse_from_file(point) hv_val = int(meta['external_meta']['HV1_value']) if hv_val not in counts_madc: counts_madc[hv_val] = [] amps = parse_madc_binary(data) filt = np.logical_and(amps > threshold_madc, amps < threshold_madc_h) counts_madc[hv_val].append(amps[filt].size) counts = {} for point in points: _, meta, data = dfparser.parse_from_file(point) hv_val = int(meta['external_meta']['HV1_value']) time = meta['params']['b_size'] * meta['params']['events_num'] / \ meta['params']['sample_freq'] point_ds = dfparser.Point() point_ds.ParseFromString(data) amps = np.hstack([ list(block.events.amplitudes) for block in point_ds.channels[0].blocks ]) if hv_val not in counts: counts[hv_val] = [] filt = np.logical_and(amps > threshold, amps < threshold_h) counts[hv_val].append(float(amps[filt].size) / time * 30.0) counts = {key: np.mean(counts[key]) for key in counts} counts_madc = {key: np.mean(counts_madc[key]) for key in counts_madc} counts_x = list(counts.keys()) counts_y = [counts[key] for key in counts.keys()] counts_madc_x = list(counts_madc.keys()) counts_madc_y = [counts_madc[key] for key in counts_madc.keys()] fig, axes = plt.subplots() fig.canvas.set_window_title("energy_spectrum_compare") fig.suptitle("CAMAC MADC vs. Lan10-12PCI energy spectrums \n" "Set - %s" % (set_path)) axes.plot(counts_x, counts_y, 'ro', label='Lan10-12PCI') axes.plot(counts_madc_x, counts_madc_y, 'bo', label='MADC') axes.set_xlabel("Voltage, V") axes.set_ylabel("Efficient counts") axes.legend() print(counts_x) print(counts_madc_x) # assert counts_x == counts_madc_x fig, axes = plt.subplots() fig.canvas.set_window_title("energy_spectrum_compare_ratio") fig.suptitle("CAMAC MADC vs. Lan10-12PCI energy spectrums ratio \n" "Set - %s" % (set_path)) axes.plot(counts_x, 1 - np.array(counts_y) / np.array(counts_madc_y), 'ro', label='') axes.set_xlabel("Voltage, V") axes.set_ylabel("MADS vs Lan10_12PCI ratio")
def main(): """Compare Lan10-12PCI points energy spectrum for several points.""" seaborn.set_context("poster") data = np.genfromtxt("/home/chernov/Downloads/set_43_detector.out") x_data = data[:, 0] y_points = data[:, 1:].transpose() y_points = (y_points.T / np.max(y_points, axis=1)).T df_data_root = "/home/chernov/data_processed" points_path = [ "2017_05/Fill_3/set_43/p0(30s)(HV1=16000).df", "2017_05/Fill_3/set_43/p36(30s)(HV1=17000).df", "2017_05/Fill_3/set_43/p80(30s)(HV1=15000).df", "2017_05/Fill_3/set_43/p102(30s)(HV1=14000).df" ] bins = 500 range_ = (0, 8000) for idx, point_rel in enumerate(points_path): _, _, data = dfparser.parse_from_file( path.join(df_data_root, point_rel)) point = dfparser.Point() point.ParseFromString(data) amps = np.hstack([ list(block.events.amplitudes) for block in point.channels[0].blocks ]) hist, x_point = np.histogram(amps, bins=bins, range=range_) hist = hist / np.max(hist[np.where(x_point > 3000)[0][0]:]) func = interp1d(x_point[1:] + x_point[:-1], hist, bounds_error=False, fill_value=0) func_mod = lambda x, a, b, c: c * func(a * x + b) x_peak = np.where(np.logical_and(x_data > 1000, x_data < 1600)) popt, _ = curve_fit(func_mod, x_data[x_peak], y_points[idx][x_peak], p0=[3.68, 700, 1]) fig, axes = plt.subplots() fig.canvas.set_window_title(point_rel) fig.suptitle("CAMAC MADC vs. Lan10-12PCI spectrums") axes.set_title("File - %s. \nOptimized parameters: a=%s, b=%s, c=%s" % (point_rel, *np.round(popt, 2))) axes.set_xlabel("Bins, ch") axes.set_xlim(0, 2000) # axes.set_yscale("log", nonposx='clip') x_interp = np.linspace(0, 2000, 500) axes.plot(x_interp, func_mod(x_interp, *popt), label="Lan10-12PCI") axes.plot(x_data, y_points[idx], label="CAMAC MADC") axes.legend()