def detect_anomalies_with_rolling_mean(ts, num_stds, window, verbose): """Detect outliers in the wheel RPM data by comparing points against [num_stds] standard deviations from a rolling mean. Inputs: ts [pd Series]: A pandas Series with a DatetimeIndex and a column for RPM. num_stds [float]: The number of standard deviations away from the mean used to define point outliers. window [int]: Window size; the number of samples to include in the rolling mean. verbose [bool]: When True, a plot of the rolling mean will be displayed before outliers are detected. Optional Inputs: None Outputs: time_series_with_outliers [pd DataFrame]: A pandas DataFrame with a DatetimeIndex, and columns for RPM (real values) and Outlier (True or False). outliers [pd Series]: The detected outliers, as a pandas Series with a DatetimeIndex and a column for the outlier value. Optional Outputs: None Example: wheel_rpm_with_outliers, outliers = detect_anomalies_with_rolling_mean(time_series, 2, window, False) """ if window <= 0: raise ValueError( '\'window\' must be given a value greater than 0 when using rolling mean.' ) else: # Gather statistics in preparation for outlier detection rolling_mean = ts.rolling(window=window, center=False).mean() first_window_mean = ts.iloc[:window].mean() for i in range( window ): # fill first 'window' samples with mean of those samples rolling_mean[i] = first_window_mean std = float(ts.values.std(ddof=0)) X = ts.values outliers = pd.Series() time_series_with_outliers = pd.DataFrame({'RPM': ts}) time_series_with_outliers['Outlier'] = 'False' if verbose: pyplot.plot(ts, color='blue', label='Time Series') pyplot.plot(rolling_mean, color='black', label='Rolling Mean') pyplot.legend(loc='best') pyplot.title('Time Series & Rolling Mean') pyplot.show() # Start a progress bar widgets = [ progressbar.Percentage(), progressbar.Bar(), progressbar.Timer(), ' ', progressbar.AdaptiveETA() ] progress_bar_sliding_window = progressbar.ProgressBar( widgets=[progressbar.FormatLabel('Wheel RPM Outliers ')] + widgets, max_value=int(len(X))).start() # Label outliers using standard deviation for t in range(len(X)): obs = X[t] y = rolling_mean[t] if abs(y - obs) > std * num_stds: time_series_with_outliers.at[ts.index[t], 'Outlier'] = 'True' outlier = pd.Series(obs, index=[ts.index[t]]) outliers = outliers.append(outlier) progress_bar_sliding_window.update(t) # advance progress bar return time_series_with_outliers, outliers
valid = True train = selection[:args.n // 2] test = selection[args.n // 2:] print("Selected {} moves for training, {} for testing".format( len(train), len(test))) import shogi print("--------------- Caching Legal Moves ----------------") widgets = [ ' [', progressbar.Timer(), '] ', progressbar.Bar(), ' (', progressbar.AdaptiveETA(), ') ', progressbar.Percentage() ] bar = progressbar.ProgressBar(maxval=len(selection), widgets=widgets) completed_games = 0 total_positions = 0 # Cache to store all of the legal moves for all the N sample positions legal_moves_cache = dict() for game in selection: pos, move = game[0], game[1] board = shogi.Board(pos) actions = [] for move in board.legal_moves: # Make the move and record the str representation of the new position=
def makeProgressBar(): return progressbar.ProgressBar(widgets=[ progressbar.Bar(), ' ', progressbar.Percentage(), ' ', progressbar.AdaptiveETA() ])
def detectStealthCMEsInSOHO(output_path='/Users/shawnpolson/Pictures/stealth_cmes_real_window/'): soho = pd.read_csv('/Users/shawnpolson/Documents/School/Spring 2018/Data Mining/StealthCMEs/savesets/Historical CME Data (SOHO).csv') print(soho.head()) #goes_data = readsav('/Users/tyleralbee/Desktop/GoesEventsMegsAEra.sav') #1. Add new column to SOHO catalog named "Stealth?" sLength = len(soho['Time']) # number of rows in the SOHO catalog soho['Stealth?'] = pd.Series(np.nan, index=soho.index) # add the new column to the catalog csv_filename = output_path + 'stealth_soho_catalog_best_window_cont_{0}.csv'.format(Time.now().iso) soho.to_csv(csv_filename, header=True, index=False, mode='w') # Start a progress bar widgets = [progressbar.Percentage(), progressbar.Bar(), progressbar.Timer(), ' ', progressbar.AdaptiveETA()] progress_bar_sliding_window = progressbar.ProgressBar( widgets=[progressbar.FormatLabel('Stealth CME Search ')] + widgets, max_value=int(sLength)).start() #2. Set up loop through SOHO catalog for i in range(0, int(sLength)): #3. For each row, pull date and time from row soho_row = soho.iloc[i] date = soho_row['Date'] time = soho_row['Time'] timeObj = pd.to_datetime(date + 'T' + time) #4. Convert that date/time to a sunpy.time.TimeRange (will need to decide how wide of a time window) # From a CME researcher: "an event is likely correlated if the CME SOHO/LASCO start time is between 2 hours before # and 4 hours after the dimming start time and within 45 degrees of the dimming location # (I convert dimming central coordinate to PA and compare that with CME PA)”. # TODO: check angles time_range = TimeRange(timeObj - timedelta(hours=4), timeObj + timedelta(hours=2)) # per the above comment #5. Send TimeRange to sunpy.instr.goes.get_goes_event_list(timerange, goes_class_filter=None) flares = sunpy.instr.goes.get_goes_event_list(time_range, goes_class_filter=None) #print(flares) #6. If that returns an empty list, add 'yes' to row['Stealth?'] if (len(flares) == 0): soho_row['Stealth?'] = 'yes' soho.iloc[i] = soho_row #7. Else, add 'no' to row['Stealth?'] else: soho_row['Stealth?'] = 'no' soho.iloc[i] = soho_row csv_filename = output_path + 'stealth_soho_catalog_best_window_cont_{0}.csv'.format(Time.now().iso) soho.to_csv(csv_filename, header=True, index=False, mode='w') progress_bar_sliding_window.update(i) # advance progress bar csv_filename = output_path + 'stealth_soho_catalog_best_window_cont_{0}.csv'.format(Time.now().iso) soho.to_csv(csv_filename, header=True, index=False, mode='w') print("Done!")
def scan(self): self.pulser_dac_parameters = self.scan_parameters.PlsrDAC self.colpr_addr_parameters = self.scan_parameters.Colpr_Addr description = np.dtype([ ('colpr_addr', np.uint32), ('PlsrDAC', np.int32), ('voltage', np.float) ]) # output data table description, native NumPy dtype data = self.raw_data_file.h5_file.create_table( self.raw_data_file.h5_file.root, name='plsr_dac_data', description=description, title='Data from PlsrDAC calibration scan') progress_bar = progressbar.ProgressBar( widgets=[ '', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA() ], maxval=len(self.pulser_dac_parameters) * len(self.colpr_addr_parameters) * self.repeat_measurements, term_width=80) progress_bar.start() progress_bar_index = 0 for colpr_address in self.colpr_addr_parameters: if self.abort_run.is_set(): break self.set_scan_parameters(Colpr_Addr=colpr_address) commands = [] commands.extend(self.register.get_commands("ConfMode")) self.register.set_global_register_value("Colpr_Addr", colpr_address) commands.extend( self.register.get_commands("WrRegister", name="Colpr_Addr")) commands.extend(self.register.get_commands("RunMode")) self.register_utils.send_commands(commands) for pulser_dac in self.pulser_dac_parameters: if self.abort_run.is_set(): break self.set_scan_parameters(PlsrDAC=pulser_dac) commands = [] commands.extend(self.register.get_commands("ConfMode")) self.register.set_global_register_value("PlsrDAC", pulser_dac) commands.extend( self.register.get_commands("WrRegister", name="PlsrDAC")) commands.extend(self.register.get_commands("RunMode")) self.register_utils.send_commands(commands) actual_data = np.zeros(shape=(self.repeat_measurements, ), dtype=description) actual_data['colpr_addr'] = colpr_address actual_data["PlsrDAC"] = pulser_dac for index, pulser_dac in enumerate( range(self.repeat_measurements)): voltage_string = self.dut['Multimeter'].get_voltage() voltage = float(voltage_string.split(',')[0]) actual_data['voltage'][index] = voltage # logging.info('Measured %.2fV', voltage) progress_bar_index += 1 progress_bar.update(progress_bar_index) # append data to HDF5 file data.append(actual_data) progress_bar.finish() data.flush()
def scan(self): # Output data structures data_array = np.zeros(shape=(len(self.scan_parameter_values), self.max_data_index - 1), dtype=np.float16) data_out = self.raw_data_file.h5_file.create_carray( self.raw_data_file.h5_file.root, name='PlsrDACwaveforms', title='Waveforms from transient PlsrDAC calibration scan', atom=tb.Atom.from_dtype(data_array.dtype), shape=data_array.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False)) data_out.attrs.scan_parameter_values = self.scan_parameter_values data_out.attrs.dimensions = ['plsrdac', 'time', 'voltage'] trigger_levels = [] progress_bar = progressbar.ProgressBar(widgets=[ '', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA() ], maxval=len( self.scan_parameter_values), term_width=80) progress_bar.start() for index, scan_parameter in enumerate(self.scan_parameter_values): # Update PlsrDAC parameter self.set_scan_parameters(**{'PlsrDAC': scan_parameter}) # tell run base self.set_scan_parameter('PlsrDAC', scan_parameter) # set in FE # Get actual high level and set trigger level to the middle self.dut['Oscilloscope'].set_acquire_mode( 'SAMPLE') # clears also averaging storage time.sleep( 1.5 ) # tektronix needs time to change mode and clear averaging storage (bad programing...) self.dut['Oscilloscope'].force_trigger() time.sleep(1.5) # give the trigger some time raw_data = self.dut['Oscilloscope'].get_data(channel=self.channel) times, voltages = interpret_oscilloscope_data(raw_data) trigger_level = (np.amax(voltages) - np.amin(voltages) ) / 2. + self.trigger_level_offset * 1e-3 self.dut['Oscilloscope'].set_trigger_level(trigger_level) if self.show_debug_plots: plt.clf() plt.grid() plt.plot(times * 1e9, voltages * 1e3, label='Data') plt.plot(times * 1e9, np.repeat([trigger_level * 1e3], len(times)), '--', label='Trigger (%d mV)' % (trigger_level * 1000)) plt.xlabel('Time [ns]') plt.ylabel('Voltage [mV]') plt.legend(loc=0) plt.show() # Setup data aquisition and start scan loop self.dut['Oscilloscope'].set_acquire_mode( 'AVERAGE' ) # average to get rid of noise and keeping high band width time.sleep( 1.5) # tektronix needs time to change mode (bad programing...) super(PlsrDacTransientCalibration, self).scan() # analog scan loop raw_data = self.dut['Oscilloscope'].get_data(channel=self.channel) times, voltages = interpret_oscilloscope_data(raw_data) data_array[index, :] = voltages[:self.max_data_index] trigger_levels.append( float(self.dut['Oscilloscope'].get_trigger_level())) progress_bar.update(index) if self.show_debug_plots: plt.clf() plt.ylim(0, 1500) plt.grid() plt.plot(times * 1e9, voltages * 1e3, label='Data') plt.plot(times * 1e9, np.repeat([trigger_level * 1e3], len(times)), '--', label='Trigger (%d mV)' % (trigger_level * 1000)) plt.xlabel('Time [ns]') plt.ylabel('Voltage [mV]') plt.legend(loc=0) plt.show() data_out[:] = data_array data_out.attrs.trigger_levels = trigger_levels data_out.attrs.times = times.tolist() progress_bar.finish()
def load_data(self, show_warnings, show_progressbar): self.all_errors = {} self.format_fields = {} m = Manager() q = m.Queue() q_bar = Manager().Queue() idxs = range(len(self.found_logs)) if show_progressbar: result = ProgressPool([(process_files, "{}".format(self.found_logs[i]), [i, self.found_logs, self.log_files_format, self.directory, self.time_zones, self.positions, q, q_bar, self.additive_link, self.user_events, self.user_hosts, self.time_ranges, self.user_vms, self.vm_timeline, self.subtasks, self.needed_lines, self.real_line_num, [mes['flow_id'] for l in self.vm_tasks.keys() for t in self.vm_tasks[l].keys() for mes in (self.vm_tasks)[l][t] if ('flow_id' in mes.keys() and mes['flow_id'] != '')], show_warnings]) for i in idxs], processes=4) else: result = [] run_args = [[i, self.found_logs, self.log_files_format, self.directory, self.time_zones, self.positions, q, q_bar, self.additive_link, self.user_events, self.user_hosts, self.time_ranges, self.user_vms, self.vm_timeline, self.subtasks, self.needed_lines, self.real_line_num, [mes['flow_id'] for l in self.vm_tasks.keys() for t in self.vm_tasks[l].keys() for mes in (self.vm_tasks)[l][t] if ('flow_id' in mes.keys() and mes['flow_id'] != '')], show_warnings] for i in idxs] widget_style = ['Load: ', progressbar.Percentage(), ' (', progressbar.SimpleProgress(), ')', ' ', progressbar.Bar(), ' ', progressbar.Timer(), ' ', progressbar.AdaptiveETA()] sum_lines = [] for log in self.found_logs: sum_lines += [p[1] - p[0] for p in self.positions[log]] sum_lines = sum(sum_lines) bar = ProgressBar(widgets=widget_style, max_value=sum_lines) pos = 0 with Pool(processes=4) as pool: worker = pool.imap(star, run_args) while True: try: try: while True: result += [worker.next(0)] except multiprocessing.TimeoutError: pass while not q_bar.empty(): pos_tmp, name = q_bar.get() pos += pos_tmp bar.update(pos) except StopIteration: break bar.finish() for idx, log in enumerate(self.found_logs): self.all_errors[log] = result[idx][0] # saving logfile format fields names self.format_fields[log] = result[idx][1] while not q.empty(): warn = q.get() self.out_descr.write(warn) if (self.all_errors == {} or all([self.all_errors[l] == [] for l in self.all_errors.keys()])): self.out_descr.write('No matches.\n') exit()
def create_hit_table( input_file_name, tdc_calibation_file, plsr_dac_calibation_file, n_sub_files=8 ): # loops over all root files and merges the data into a hdf5 file aligned at the event number print 'Converting data from CERN ROOT TTree to hdf5 table' charge_calibration_values, tdc_calibration, tdc_error, tot_calibration, tot_error = get_charge_calibration( tdc_calibation_file, plsr_dac_calibation_file) # add all files that have the input_file_name praefix and load their data input_file_names = [ input_file_name + '_t%d.root' % index for index in range(n_sub_files) if os.path.isfile(input_file_name + '_t%d.root' % index) ] n_files = len(input_file_names) input_files_root = [ r.TFile(file_name, 'read') for file_name in input_file_names ] pixel_digits = [ input_file_root.Get('EventData').Get('Pixel Digits') for input_file_root in input_files_root ] n_hits = [pixel_digit.GetEntries() for pixel_digit in pixel_digits] # total pixel hits to analyze n_total_hits = sum(n_hits) with tb.open_file(input_file_name + '_interpreted.h5', 'w') as out_file_h5: hit_table = out_file_h5.create_table( out_file_h5.root, name='Hits', description=data_struct.HitInfoTable, title='hit_data', filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False)) # tmp data structures to be filles by ROOT data = {} for index, pixel_digit in enumerate(pixel_digits): column_data = {} for branch in pixel_digit.GetListOfBranches( ): # loop over the branches column_data[branch.GetName()] = np.zeros(shape=1, dtype=np.int32) branch.SetAddress(column_data[branch.GetName()].data) data[index] = column_data # result data structur to be filles in the following loop hits = np.zeros((n_total_hits, ), dtype=tb.dtype_from_descr(data_struct.HitInfoTable)) # get file index with lowest event number for pixel_digit in pixel_digits: pixel_digit.GetEntry(0) min_event_number = min( [data[index]['event'][0] for index in range(n_files)]) actual_file_index = np.where( np.array([data[index]['event'][0] for index in range(n_files)]) == min_event_number)[0][0] indices = [0] * n_files table_index = 0 actual_data = data[actual_file_index] actual_event_number = actual_data['event'][0] last_valid_event_number = 0 last_tdc = 0 expected_event_number = actual_event_number indices[actual_file_index] = 1 progress_bar = progressbar.ProgressBar(widgets=[ '', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA() ], maxval=n_total_hits, term_width=80) progress_bar.start() def add_actual_data(actual_data, table_index): if actual_data['column'] >= 0 and actual_data[ 'column'] < 80 and actual_data['row'] >= 0 and actual_data[ 'row'] < 336: tdc_interpolation = interp1d( x=charge_calibration_values, y=tdc_calibration[actual_data['column'], actual_data['row']], kind='slinear', bounds_error=False, fill_value=0) tdc = tdc_interpolation(actual_data['charge']) tot_interpolation = interp1d( x=charge_calibration_values, y=tot_calibration[actual_data['column'], actual_data['row']], kind='slinear', bounds_error=False, fill_value=0) tot = tot_interpolation(actual_data['charge']) if math.isnan( tdc ): # do not add hits where tdc is nan, these pixel have a very high threshold or do not work return table_index if tdc == 0 and actual_data[ 'charge'] > 10000: # no calibration for TDC due to high charge, thus mark as TDC overflow event hits[table_index]['event_status'] |= 0b0000010000000000 tdc = 4095 if tot == 0 and actual_data[ 'charge'] > 10000: # no calibration for TOT due to high charge, thus set max tot tot = 13 hits[table_index]['event_status'] |= 0b0000000100000000 hits[table_index]['event_number'] = actual_data['event'][ 0].astype(np.int64) hits[table_index]['column'] = (actual_data['column'] + 1).astype(np.uint8) hits[table_index]['row'] = (actual_data['row'] + 1).astype( np.uint16) hits[table_index]['TDC'] = int(actual_data['charge'] / 300.) hits[table_index]['tot'] = int(tot) table_index += 1 return table_index while True: actual_event_number = actual_data['event'][0] if (actual_event_number == expected_event_number or actual_event_number == expected_event_number - 1): # check if event number increases actual_index, actual_digits, actual_data = indices[ actual_file_index], pixel_digits[actual_file_index], data[ actual_file_index] table_index = add_actual_data(actual_data, table_index) else: # event number does not increase, thus the events are in another file --> switch file or the event number is missing file_event_numbers = [ data[file_index]['event'][0] for file_index in range(n_files) ] # all files actual event number actual_file_index = np.where( file_event_numbers == min(file_event_numbers))[0][0] actual_index, actual_digits, actual_data = indices[ actual_file_index], pixel_digits[actual_file_index], data[ actual_file_index] actual_event_number = actual_data['event'][0] table_index = add_actual_data(actual_data, table_index) progress_bar.update(table_index) expected_event_number = actual_event_number + 1 actual_digits.GetEntry(actual_index) if indices[actual_file_index] < n_hits[ actual_file_index]: # simply stop when the first file is fully iterated indices[actual_file_index] += 1 else: break # Set missing data and store to file hits[:table_index]['LVL1ID'] = hits[:table_index]['event_number'] % 255 hits[:table_index]['BCID'] = hits[:table_index]['LVL1ID'] hits[:table_index]['relative_BCID'] = 6 hit_table.append(hits[:table_index]) progress_bar.finish() for input_file_root in input_files_root: input_file_root.Close()
def get_signal_planar_sensor(x_track, q_in, D, S, n_eff_0, is_ntype, is_oxygenated, V_bias, fluence, temperatur, t, dt, N): ''' Function to return the signal of a planar silicon sensor as a function of the time. The parameters are: x_track: offset from the electrode mean [um] q_in: deposited charge [e-h pairs] D: sensor thickness [um] S: electrode width [um] n_eff_0: doping concentration without fluence [10^12 Neq/cm^3] isNtype: = 1 for ntype sensor isOxygenated: = 1 for oxygenated sensor V_bias: bias of the sensor [V] fluence: radiation dose [10^12 Neq/cm^3] temperatur: temerature [K] t: time points of simulation [ns] dt: time step of simulation [ns] CHOOSE SMALL ENOUGH! N: number of quasi particles along the track ''' # Constants Q_e = constants.e # Electron charge [C] e_h = q_in # deposited charge in electron hole pairs x_0 = x_track # offset from the middle of the sensor pixel (x-direction) [um] n_eff = get_eff_acceptor_concentration(fluence, n_eff_0, is_ntype, is_oxygenated) if fluence: tr_e = get_trapping(fluence * 1e12, is_electron=True, paper=2) # [ns] tr_h = get_trapping(fluence * 1e12, is_electron=False, paper=2) # [ns] e_h = e_h / N # Start positions of the electrons if N == 1: y_e = np.atleast_1d(D / 2.) else: y_e = np.linspace(0, D, N) y_h = y_e.copy() # start positions of the electron holes x_e = np.ones_like(y_h) * x_0 # Positions of the electrons in x x_h = x_e.copy() # Positions of the electron holes in x Q_ind_e = np.zeros_like(y_e) # start induced charge from electrons Q_ind_h = np.zeros_like(y_e) # start induced charge from holes Q_ind_e_vec = np.zeros(shape=(y_e.shape[0], t.shape[0])) Q_ind_h_vec = np.zeros(shape=(y_e.shape[0], t.shape[0])) y_e_vec = np.zeros(shape=(y_e.shape[0], t.shape[0])) y_h_vec = np.zeros(shape=(y_e.shape[0], t.shape[0])) v_e_vec = np.zeros(shape=(y_e.shape[0], t.shape[0])) v_h_vec = np.zeros(shape=(y_e.shape[0], t.shape[0])) E_q_y_e_vec = np.zeros(shape=(y_e.shape[0], t.shape[0])) E_q_y_h_vec = np.zeros(shape=(y_e.shape[0], t.shape[0])) E_w_y_e_vec = np.zeros(shape=(y_e.shape[0], t.shape[0])) E_w_y_h_vec = np.zeros(shape=(y_e.shape[0], t.shape[0])) Phi_w_y_e_vec = np.zeros(shape=(y_e.shape[0], t.shape[0])) Phi_w_y_h_vec = np.zeros(shape=(y_e.shape[0], t.shape[0])) progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=len(t), term_width=80) progress_bar.start() for index, i in enumerate(t): # time loop # Electric, weighting field a charge carrier positions # Electrons: _, E_q_y_e = get_field(x=x_e, # Electric field [V/um] y=y_e, V_bias=V_bias, n_eff=n_eff, D=D, S=None, # FIXME is_planar=True) Phi_w_e = get_weighting_potential(x=x_e, y=y_e, D=D, S=S, is_planar=True) _, E_w_y_e = get_weighting_field(x=x_e, y=y_e, D=D, S=S, is_planar=True) # Holes _, E_q_y_h = get_field(x=x_h, # Electric field [V/um] y=y_h, V_bias=V_bias, n_eff=n_eff, D=D, S=None, # FIXME is_planar=True) Phi_w_h = get_weighting_potential(x=x_h, y=y_h, D=D, S=S, is_planar=True) _, E_w_y_h = get_weighting_field(x=x_h, y=y_h, D=D, S=S, is_planar=True) # Movement: # Electrons v_e = - E_q_y_e * get_mobility(E_q_y_e * 1e5, # Velocity [um/ns] temperature=temperatur, is_electron=True) dy_e = v_e * dt y_e = y_e + dy_e # Boundaries e_in_boundary = np.logical_and(y_e <= D, y_e >= 0) E_w_y_e[~e_in_boundary] = 0 E_q_y_e[~e_in_boundary] = 0 v_e[~e_in_boundary] = 0 y_e[~e_in_boundary] = D E_w_y_e[~e_in_boundary] = 0 E_q_y_e[~e_in_boundary] = 0 v_e[~e_in_boundary] = 0 y_e[~e_in_boundary] = 0 # Holes v_h = E_q_y_h * get_mobility(E_q_y_h * 1e5, # Velocity [um/ns] temperature=temperatur, is_electron=False) dy_h = v_h * dt y_h = y_h + dy_h # Boundaries h_in_boundary = np.logical_and(y_h <= D, y_h >= 0) E_w_y_h[~h_in_boundary] = 0 E_q_y_h[~h_in_boundary] = 0 v_h[~h_in_boundary] = 0 y_h[~h_in_boundary] = 0 E_w_y_h[~h_in_boundary] = 0 E_q_y_h[~h_in_boundary] = 0 v_h[~h_in_boundary] = 0 y_h[~h_in_boundary] = D # Induced charge calculation with trapping # electrons dQ_e = np.zeros_like(y_e) dQ_e[e_in_boundary] = e_h * E_w_y_e[e_in_boundary] * dy_e[e_in_boundary] if fluence: dQ_e[e_in_boundary] *= np.exp(-i / tr_e) # Holes dQ_h = np.zeros_like(y_h) dQ_h[h_in_boundary] = -e_h * E_w_y_h[h_in_boundary] * dy_h[h_in_boundary] if fluence: dQ_h[h_in_boundary] *= np.exp(-i / tr_h) # Sum up Q_ind_e += dQ_e Q_ind_h += dQ_h # Data for plotting Q_ind_e_vec[:, index] = Q_ind_e Q_ind_h_vec[:, index] = Q_ind_h y_e_vec[:, index] = y_e v_e_vec[:, index] = v_e y_h_vec[:, index] = y_h v_h_vec[:, index] = v_h E_q_y_e_vec[:, index] = E_q_y_e E_q_y_h_vec[:, index] = E_q_y_h E_w_y_e_vec[:, index] = E_w_y_e E_w_y_h_vec[:, index] = E_w_y_h Phi_w_y_e_vec[:, index] = Phi_w_e Phi_w_y_h_vec[:, index] = Phi_w_h progress_bar.update(index) progress_bar.finish() # It is already summed? Q_tot_e = Q_ind_e_vec.sum(axis=0) Q_tot_h = Q_ind_h_vec.sum(axis=0) Q_tot = Q_tot_e + Q_tot_h return y_e_vec, y_h_vec, Q_tot, Q_tot_e, Q_tot_h
def create_hitor_calibration(output_filename): logging.info('Analyze and plot results of %s', output_filename) def plot_calibration(col_row_combinations, scan_parameter, calibration_data, filename): # Result calibration plot function for index, (column, row) in enumerate(col_row_combinations): logging.info("Plot calibration for pixel " + str(column) + '/' + str(row)) fig = Figure() FigureCanvas(fig) ax = fig.add_subplot(111) fig.patch.set_facecolor('white') ax.grid(True) ax.errorbar(scan_parameter, calibration_data[column - 1, row - 1, :, 0] * 25. + 25., yerr=[calibration_data[column - 1, row - 1, :, 2] * 25, calibration_data[column - 1, row - 1, :, 2] * 25], fmt='o', label='FE-I4 ToT [ns]') ax.errorbar(scan_parameter, calibration_data[column - 1, row - 1, :, 1] * 1.5625, yerr=[calibration_data[column - 1, row - 1, :, 3] * 1.5625, calibration_data[column - 1, row - 1, :, 3] * 1.5625], fmt='o', label='TDC ToT [ns]') ax.set_title('Calibration for pixel ' + str(column) + '/' + str(row)) ax.set_xlabel('Charge [PlsrDAC]') ax.set_ylabel('TOT') ax.legend(loc=0) filename.savefig(fig) if index > 100: # stop for too many plots logging.info('Do not create pixel plots for more than 100 pixels to safe time') break with AnalyzeRawData(raw_data_file=output_filename, create_pdf=True) as analyze_raw_data: # Interpret the raw data file analyze_raw_data.create_occupancy_hist = False # too many scan parameters to do in ram histograming analyze_raw_data.create_hit_table = True analyze_raw_data.create_tdc_hist = True analyze_raw_data.align_at_tdc = True # align events at TDC words, first word of event has to be a tdc word analyze_raw_data.interpret_word_table() analyze_raw_data.interpreter.print_summary() analyze_raw_data.plot_histograms() n_injections = analyze_raw_data.n_injections # store number of injections for later cross check with tb.open_file(output_filename + '_interpreted.h5', 'r') as in_file_h5: # Get scan parameters from interpreted file meta_data = in_file_h5.root.meta_data[:] hits = in_file_h5.root.Hits[:] scan_parameters_dict = get_scan_parameter(meta_data) inner_loop_parameter_values = scan_parameters_dict[next(reversed(scan_parameters_dict))] # inner loop parameter name is unknown scan_parameter_names = scan_parameters_dict.keys() col_row_combinations = get_unique_scan_parameter_combinations(in_file_h5.root.meta_data[:], scan_parameters=('column', 'row'), scan_parameter_columns_only=True) meta_data_table_at_scan_parameter = get_unique_scan_parameter_combinations(meta_data, scan_parameters=scan_parameter_names) parameter_values = get_scan_parameters_table_from_meta_data(meta_data_table_at_scan_parameter, scan_parameter_names) event_number_ranges = get_ranges_from_array(meta_data_table_at_scan_parameter['event_number']) event_ranges_per_parameter = np.column_stack((parameter_values, event_number_ranges)) event_numbers = hits['event_number'].copy() # create contigous array, otherwise np.searchsorted too slow, http://stackoverflow.com/questions/15139299/performance-of-numpy-searchsorted-is-poor-on-structured-arrays with tb.openFile(output_filename + "_calibration.h5", mode="w") as calibration_data_file: logging.info('Create calibration') output_pdf = PdfPages(output_filename + "_calibration.pdf") calibration_data = np.zeros(shape=(80, 336, len(inner_loop_parameter_values), 4), dtype='f4') # result of the calibration is a histogram with col_index, row_index, plsrDAC value, mean discrete tot, rms discrete tot, mean tot from TDC, rms tot from TDC progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=len(event_ranges_per_parameter), term_width=80) progress_bar.start() for index, (parameter_values, event_start, event_stop) in enumerate(event_ranges_per_parameter): if event_stop is None: # happens for the last chunk event_stop = hits[-1]['event_number'] array_index = np.searchsorted(event_numbers, np.array([event_start, event_stop])) actual_hits = hits[array_index[0]:array_index[1]] actual_col, actual_row, parameter_value = parameter_values if len(hits[np.logical_and(actual_hits['column'] != actual_col, actual_hits['row'] != actual_row)]): logging.warning('There are %d hits from not selected pixels in the data', len(actual_hits[np.logical_and(actual_hits['column'] != actual_col, actual_hits['row'] != actual_row)])) actual_hits = actual_hits[np.logical_and(actual_hits['column'] == actual_col, actual_hits['row'] == actual_row)] actual_tdc_hits = actual_hits[(actual_hits['event_status'] & 0b0000111110011100) == 0b0000000100000000] # only take hits from good events (one TDC word only, no error) actual_tot_hits = actual_hits[(actual_hits['event_status'] & 0b0000100010011100) == 0b0000000000000000] # only take hits from good events for tot tot, tdc = actual_tot_hits['tot'], actual_tdc_hits['TDC'] if tdc.shape[0] != n_injections and index == event_ranges_per_parameter.shape[0] - 1: logging.warning('There are %d != %d TDC hits for %s = %s', tdc.shape[0], n_injections, str(scan_parameter_names), str(parameter_values)) inner_loop_scan_parameter_index = np.where(parameter_value == inner_loop_parameter_values)[0][0] # translate the scan parameter value to an index for the result histogram calibration_data[actual_col - 1, actual_row - 1, inner_loop_scan_parameter_index, 0] = np.mean(tot) calibration_data[actual_col - 1, actual_row - 1, inner_loop_scan_parameter_index, 1] = np.mean(tdc) calibration_data[actual_col - 1, actual_row - 1, inner_loop_scan_parameter_index, 2] = np.std(tot) calibration_data[actual_col - 1, actual_row - 1, inner_loop_scan_parameter_index, 3] = np.std(tdc) progress_bar.update(index) calibration_data_out = calibration_data_file.createCArray(calibration_data_file.root, name='HitOrCalibration', title='Hit OR calibration data', atom=tb.Atom.from_dtype(calibration_data.dtype), shape=calibration_data.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False)) calibration_data_out[:] = calibration_data calibration_data_out.attrs.dimensions = scan_parameter_names calibration_data_out.attrs.scan_parameter_values = inner_loop_parameter_values plot_calibration(col_row_combinations, scan_parameter=inner_loop_parameter_values, calibration_data=calibration_data, filename=output_pdf) output_pdf.close() progress_bar.finish()
def WeatherGraphs(GraphPath, Weather_Data): """Creates monthly weather graphs Creates monthly graphs showing the weather patterns Parameters ---------- GraphPath : str Path to save the graphs. Weather_Data : array-like The weather data to graph. """ majorTick = mdates.DayLocator(interval=7) # every month majorFmt = mdates.DateFormatter('%d/%m') minorTick = mdates.DayLocator() # every day if not os.path.isdir(GraphPath + 'Monthly/'): print('Monthly folder not found, creating it') os.makedirs(GraphPath + 'Monthly') if not os.path.isfile(GraphPath + 'Monthly/Weather2018-3.png'): widgets = [ 'Creating monthly weather graph: ', progressbar.Percentage(), ' ', progressbar.Bar(marker='∎', left='[', right=']'), ' ', progressbar.AdaptiveETA() ] pbar = progressbar.ProgressBar(widgets=widgets, maxval=60) pbar.start() month = 3 year = 3 count = 0 done = False #print('Creating Monthly weather graph [%]') while not done: pbar.update(count) count = count + 1 #if count % 6 == 0: # print('\u220E', end='', flush=True) StartPoint = Weather_Data.index.get_loc( '201' + str(year) + '-' + str(month) + '-' + '01 00:00', method='nearest') EndPoint = Weather_Data['201' + str(year) + '-' + str(month)].shape[0] fig, ax = plt.subplots() ax.plot(Weather_Data.index[StartPoint:StartPoint + EndPoint], Weather_Data.W_speed_10['201' + str(year) + '-' + str(month)], color=Color2) # format the ticks ax.xaxis.set_major_locator(majorTick) ax.xaxis.set_major_formatter(majorFmt) ax.xaxis.set_minor_locator(minorTick) datemin = np.datetime64(Weather_Data.index[StartPoint], 'm') # round to nearest years... datemax = np.datetime64( Weather_Data.index[StartPoint + EndPoint - 1], 'm') ax.set_xlim(datemin, datemax) ax.grid(which='major', alpha=1, color='black') ax.grid(which='minor', alpha=0.05, color='black') fig.autofmt_xdate() plt.title('201' + str(year) + ' - ' + monthDict[month]) plt.ylim([0, 110]) plt.xlabel('Time [date]') plt.ylabel('Wind Speed [km/h]') plt.savefig(GraphPath + 'Monthly/' + 'Weather201' + str(year) + '-' + str(month) + '.png') plt.close(fig) if month == 12: month = 1 year = year + 1 else: month = month + 1 if year == 8 and month == 4: done = True #print('') pbar.finish()
"[ERR] Some modules failed to import. Install all the program's requirements with the following command:\npip install --upgrade -r requirements.txt" ) subprocess.call("cls" if os.name == "nt" else "clear", shell=True) print(Style.NORMAL + Fore.RESET) if os.name == "nt": print(Fore.BLUE + "Checking for / downloading FFMPEG binaries...") ff = ["ffmpeg.exe", "ffplay.exe", "ffprobe.exe"] if any([not os.path.exists(x) for x in ff]): print(Fore.GREEN) widgets = [ "Downloading FFmpeg: ", progressbar.Bar(), " ", progressbar.Percentage(), progressbar.Timer(" (%(elapsed)s elapsed, "), progressbar.AdaptiveETA(), " remaining)" ] if platform.uname()[4].lower() == "amd64": # 64-bit architecture arc = "64" mxc = 8000 # Chunks estimate else: # 32-bit architecture arc = "32" mxc = np.ceil(55159009 / 8096) # Chunks estimate url = "https://ffmpeg.zeranoe.com/builds/win{0}/static/ffmpeg-4.1-win{0}-static.zip".format( arc) fn = url.split("/")[-1] res = requests.get(url, stream=True) pb = progressbar.ProgressBar(widgets=widgets, max_value=mxc) pb.start() with open(fn, "wb") as fout:
def correlationCoefficientScan( output_path='/Users/tyleralbee/Desktop/StealthCME', eve_data_path='/Users/tyleralbee/Desktop/savesets/eve_selected_lines.csv', cme_signature='/Users/tyleralbee/Desktop/savesets/eve_lines_event_percents_fitted.csv', verbose=True): eve_lines = pd.read_csv(eve_data_path, index_col=0) eve_lines.index = pd.to_datetime(eve_lines.index) wholeDfLength = eve_lines.__len__() cme_event = pd.read_csv(cme_signature, index_col=0) cme_event.index = pd.to_datetime(cme_event.index) cmeEventLength = cme_event.__len__() if verbose: logger = JpmLogger(filename='do_correlation_coefficient_scan', path=output_path, console=True) logger.info("Starting Stealth CME search pipeline!") else: logger = None if verbose: logger.info('Loaded EVE and CME data') # Define the columns of the output catalog output_table = pd.DataFrame(columns=[ 'Event #', 'Start Time', 'End Time', 'Correlation Coefficient' ]) csv_filename = output_path + 'cc_output_{0}.csv'.format(Time.now().iso) output_table.to_csv(csv_filename, header=True, index=False, mode='w') if verbose: logger.info('Created output table definition.') # Start a progress bar widgets = [ progressbar.Percentage(), progressbar.Bar(), progressbar.Timer(), ' ', progressbar.AdaptiveETA() ] startRow = 0 endRow = cmeEventLength numSlices = int(wholeDfLength / cmeEventLength) output_row = 1 progress_bar_sliding_window = progressbar.ProgressBar( widgets=[progressbar.FormatLabel('Correlation Coefficient Analysis ') ] + widgets, max_value=numSlices).start() # ----------Loop through data set using a sliding time window------------------------------------------------------- for i in range(1, numSlices): # ----------Clip dataset to time slice window------------------------------------------------------------------- event_time_slice = eve_lines.iloc[startRow:endRow] # ---------Convert irradiance values to percentages------------------------------------------------------------- preflare_irradiance = event_time_slice.iloc[0] event_time_slice_percentages = (event_time_slice - preflare_irradiance ) / preflare_irradiance * 100.0 if verbose: logger.info( "Event {0} irradiance converted from absolute to percent units." .format(i)) # ---------Fit light curves to reduce noise--------------------------------------------------------------------- uncertainty = np.ones(len(event_time_slice_percentages) ) * 0.002545 # got this line from James's code progress_bar_fitting = progressbar.ProgressBar( widgets=[progressbar.FormatLabel('Light curve fitting: ')] + widgets, max_value=len(event_time_slice_percentages.columns)).start() for j, column in enumerate(event_time_slice_percentages): if event_time_slice_percentages[column].isnull().all().all(): if verbose: logger.info( 'Event {0} {1} fitting skipped because all irradiances are NaN.' .format(j, column)) else: eve_line_event_percentages = pd.DataFrame( event_time_slice_percentages[column]) eve_line_event_percentages.columns = ['irradiance'] eve_line_event_percentages['uncertainty'] = uncertainty fitting_path = output_path + 'Fitting/' if not os.path.exists(fitting_path): os.makedirs(fitting_path) plt.close('all') light_curve_fit, best_fit_gamma, best_fit_score = automatic_fit_light_curve( eve_line_event_percentages, plots_save_path='{0} Event {1} {2} '.format( fitting_path, j, column), verbose=verbose, logger=logger) event_time_slice_percentages[column] = light_curve_fit event_time_slice_fitted = event_time_slice_percentages # Keep our variable names explicit if verbose: logger.info('Event {0} {1} light curves fitted.'.format( j, column)) progress_bar_fitting.update(j) progress_bar_fitting.finish() if verbose: logger.info("Event {0} Light curves fitted".format(i)) # ---------Compute Correlation Coefficients--------------------------------------------------------------------- totalCorrelationCoefficient = 0.0 ds1 = event_time_slice_fitted ds2 = cme_event # Gather stats for correlation for k, column in enumerate(ds1): dsColumn1 = ds1[column] dsColumn2 = ds2[column] dsColumn1.reset_index( drop=True, inplace=True) # prevent NaNs from appearing in join dsColumn2.reset_index( drop=True, inplace=True) # prevent NaNs from appearing in join # TODO: assert that both columns have same count? n = int(dsColumn1.count()) meanA = float(dsColumn1.mean()) meanB = float(dsColumn2.mean()) stdA = float(dsColumn1.std(ddof=0)) stdB = float(dsColumn2.std(ddof=0)) # Generate correlation output dsJoined = pd.DataFrame({ 'a': dsColumn1, 'b': dsColumn2 }) # Avoids ambiguity when attr names are the same numerator = 0.0 # Stores summation of (a_i - meanA)(b_i - meanB) denominator = n * stdA * stdB for index, row in dsJoined.iterrows(): a = row['a'] b = row['b'] numerator = numerator + (a - meanA) * (b - meanB) correlationCoefficient = numerator / denominator totalCorrelationCoefficient = totalCorrelationCoefficient + correlationCoefficient # ---------Output Results--------------------------------------------------------------------------------------- eventStartTime = event_time_slice.iloc[0].name eventEndTime = event_time_slice.iloc[-1].name if not math.isnan(totalCorrelationCoefficient ) and totalCorrelationCoefficient >= 4.2: output_table.loc[output_row] = [ output_row, eventStartTime, eventEndTime, totalCorrelationCoefficient ] csv_filename = output_path + 'cc_output_{0}.csv'.format( Time.now().iso) output_table.to_csv(csv_filename, header=True, index=False, mode='w') output_row = output_row + 1 startRow = startRow + 60 # advance time window by 1 hour endRow = endRow + 60 # advance time window by 1 hour progress_bar_sliding_window.update(i) # advance progress bar
def create_threshold_calibration(scan_base_file_name, create_plots=True): # Create calibration function, can be called stand alone def analyze_raw_data_file(file_name): if os.path.isfile(os.path.splitext(file_name)[0] + '_interpreted.h5'): # skip analysis if already done logging.warning('Analyzed data file ' + file_name + ' already exists. Skip analysis for this file.') else: with AnalyzeRawData(raw_data_file=file_name, create_pdf=False) as analyze_raw_data: analyze_raw_data.create_tot_hist = False analyze_raw_data.create_tot_pixel_hist = False analyze_raw_data.create_fitted_threshold_hists = True analyze_raw_data.create_threshold_mask = True analyze_raw_data.interpreter.set_warning_output(False) # RX errors would fill the console analyze_raw_data.interpret_word_table() def store_calibration_data_as_table(out_file_h5, mean_threshold_calibration, mean_threshold_rms_calibration, threshold_calibration, parameter_values): logging.info("Storing calibration data in a table...") filter_table = tb.Filters(complib='blosc', complevel=5, fletcher32=False) mean_threshold_calib_table = out_file_h5.create_table(out_file_h5.root, name='MeanThresholdCalibration', description=data_struct.MeanThresholdCalibrationTable, title='mean_threshold_calibration', filters=filter_table) threshold_calib_table = out_file_h5.create_table(out_file_h5.root, name='ThresholdCalibration', description=data_struct.ThresholdCalibrationTable, title='threshold_calibration', filters=filter_table) for column in range(80): for row in range(336): for parameter_value_index, parameter_value in enumerate(parameter_values): threshold_calib_table.row['column'] = column threshold_calib_table.row['row'] = row threshold_calib_table.row['parameter_value'] = parameter_value threshold_calib_table.row['threshold'] = threshold_calibration[column, row, parameter_value_index] threshold_calib_table.row.append() for parameter_value_index, parameter_value in enumerate(parameter_values): mean_threshold_calib_table.row['parameter_value'] = parameter_value mean_threshold_calib_table.row['mean_threshold'] = mean_threshold_calibration[parameter_value_index] mean_threshold_calib_table.row['threshold_rms'] = mean_threshold_rms_calibration[parameter_value_index] mean_threshold_calib_table.row.append() threshold_calib_table.flush() mean_threshold_calib_table.flush() logging.info("done") def store_calibration_data_as_array(out_file_h5, mean_threshold_calibration, mean_threshold_rms_calibration, threshold_calibration, parameter_name, parameter_values): logging.info("Storing calibration data in an array...") filter_table = tb.Filters(complib='blosc', complevel=5, fletcher32=False) mean_threshold_calib_array = out_file_h5.create_carray(out_file_h5.root, name='HistThresholdMeanCalibration', atom=tb.Atom.from_dtype(mean_threshold_calibration.dtype), shape=mean_threshold_calibration.shape, title='mean_threshold_calibration', filters=filter_table) mean_threshold_calib_rms_array = out_file_h5.create_carray(out_file_h5.root, name='HistThresholdRMSCalibration', atom=tb.Atom.from_dtype(mean_threshold_calibration.dtype), shape=mean_threshold_calibration.shape, title='mean_threshold_rms_calibration', filters=filter_table) threshold_calib_array = out_file_h5.create_carray(out_file_h5.root, name='HistThresholdCalibration', atom=tb.Atom.from_dtype(threshold_calibration.dtype), shape=threshold_calibration.shape, title='threshold_calibration', filters=filter_table) mean_threshold_calib_array[:] = mean_threshold_calibration mean_threshold_calib_rms_array[:] = mean_threshold_rms_calibration threshold_calib_array[:] = threshold_calibration mean_threshold_calib_array.attrs.dimensions = ['column', 'row', parameter_name] mean_threshold_calib_rms_array.attrs.dimensions = ['column', 'row', parameter_name] threshold_calib_array.attrs.dimensions = ['column', 'row', parameter_name] mean_threshold_calib_array.attrs.scan_parameter_values = parameter_values mean_threshold_calib_rms_array.attrs.scan_parameter_values = parameter_values threshold_calib_array.attrs.scan_parameter_values = parameter_values logging.info("done") def mask_columns(pixel_array, ignore_columns): idx = np.array(ignore_columns) - 1 # from FE to Array columns m = np.zeros_like(pixel_array) m[:, idx] = 1 return np.ma.masked_array(pixel_array, m) raw_data_files = analysis_utils.get_data_file_names_from_scan_base(scan_base_file_name) first_scan_base_file_name = scan_base_file_name if isinstance(scan_base_file_name, basestring) else scan_base_file_name[0] # multilpe scan_base_file_names for multiple runs with tb.open_file(first_scan_base_file_name + '.h5', mode="r") as in_file_h5: # deduce scan parameters from the first (and often only) scan base file name ignore_columns = in_file_h5.root.configuration.run_conf[:][np.where(in_file_h5.root.configuration.run_conf[:]['name'] == 'ignore_columns')]['value'][0] parameter_name = in_file_h5.root.configuration.run_conf[:][np.where(in_file_h5.root.configuration.run_conf[:]['name'] == 'scan_parameters')]['value'][0] ignore_columns = ast.literal_eval(ignore_columns) parameter_name = ast.literal_eval(parameter_name)[1][0] calibration_file = first_scan_base_file_name + '_calibration' for raw_data_file in raw_data_files: # analyze each raw data file, not using multithreading here, it is already used in s-curve fit analyze_raw_data_file(raw_data_file) files_per_parameter = analysis_utils.get_parameter_value_from_file_names([os.path.splitext(file_name)[0] + '_interpreted.h5' for file_name in raw_data_files], parameter_name, unique=True, sort=True) logging.info("Create calibration from data") mean_threshold_calibration = np.empty(shape=(len(raw_data_files),), dtype='<f8') mean_threshold_rms_calibration = np.empty(shape=(len(raw_data_files),), dtype='<f8') threshold_calibration = np.empty(shape=(80, 336, len(raw_data_files)), dtype='<f8') if create_plots: logging.info('Saving calibration plots in: %s', calibration_file + '.pdf') output_pdf = PdfPages(calibration_file + '.pdf') progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=len(files_per_parameter.items()), term_width=80) progress_bar.start() parameter_values = [] for index, (analyzed_data_file, parameters) in enumerate(files_per_parameter.items()): parameter_values.append(parameters.values()[0][0]) with tb.open_file(analyzed_data_file, mode="r") as in_file_h5: occupancy_masked = mask_columns(pixel_array=in_file_h5.root.HistOcc[:], ignore_columns=ignore_columns) # mask the not scanned columns for analysis and plotting thresholds_masked = mask_columns(pixel_array=in_file_h5.root.HistThresholdFitted[:], ignore_columns=ignore_columns) if create_plots: plot_three_way(hist=thresholds_masked, title='Threshold Fitted for ' + parameters.keys()[0] + ' = ' + str(parameters.values()[0][0]), filename=output_pdf) plsr_dacs = analysis_utils.get_scan_parameter(meta_data_array=in_file_h5.root.meta_data[:])['PlsrDAC'] plot_scurves(occupancy_hist=occupancy_masked, scan_parameters=plsr_dacs, scan_parameter_name='PlsrDAC', filename=output_pdf) # fill the calibration data arrays mean_threshold_calibration[index] = np.ma.mean(thresholds_masked) mean_threshold_rms_calibration[index] = np.ma.std(thresholds_masked) threshold_calibration[:, :, index] = thresholds_masked.T progress_bar.update(index) progress_bar.finish() with tb.open_file(calibration_file + '.h5', mode="w") as out_file_h5: store_calibration_data_as_array(out_file_h5=out_file_h5, mean_threshold_calibration=mean_threshold_calibration, mean_threshold_rms_calibration=mean_threshold_rms_calibration, threshold_calibration=threshold_calibration, parameter_name=parameter_name, parameter_values=parameter_values) store_calibration_data_as_table(out_file_h5=out_file_h5, mean_threshold_calibration=mean_threshold_calibration, mean_threshold_rms_calibration=mean_threshold_rms_calibration, threshold_calibration=threshold_calibration, parameter_values=parameter_values) if create_plots: plot_scatter(x=parameter_values, y=mean_threshold_calibration, title='Threshold calibration', x_label=parameter_name, y_label='Mean threshold', log_x=False, filename=output_pdf) plot_scatter(x=parameter_values, y=mean_threshold_calibration, title='Threshold calibration', x_label=parameter_name, y_label='Mean threshold', log_x=True, filename=output_pdf) output_pdf.close()
def create_progressbar_reader(reader, max_reads=None, mag_format=None): """Wrap an iterable in a ProgressBar. Args: max_reads: Max number of items, if known in advance. mag_format: Function that formats an integer as a string with magnitude (e.g. 1000000 => 1M). """ import progressbar import progressbar.widgets class ProgressBarReader(progressbar.ProgressBar): """Extension of ProgressBar that supports starting and stopping the BatchReader. """ def __init__(self, iterable, widgets, max_value=None): super(ProgressBarReader, self).__init__(widgets=widgets, max_value=max_value or progressbar.UnknownLength) self._iterable = iterable self.done = False def __next__(self): try: value = next(self._iterable) if self.start_time is None: self.start() self.update(self.value + value[0]["size"]) return value except StopIteration: self.close() raise def close(self): """Finish the progress bar and close the underlying iterator. """ if not self.done: self.finish() self.done = True try: self._iterable.close() except: pass class MagCounter(progressbar.widgets.WidgetBase): """Custom widget that formats the value using a specified magnitude. """ def __init__(self, mag_format): super().__init__() self._format = mag_format def __call__(self, progress, data): return self._format(data["value"]) if max_reads: reader = ProgressBarReader(reader, [ MagCounter(mag_format), " Reads (", progressbar.Percentage(), ") ", progressbar.Timer(), " ", progressbar.Bar(), progressbar.AdaptiveETA() ], max_reads) else: reader = ProgressBarReader(reader, [ MagCounter(mag_format), " Reads", progressbar.Timer(), progressbar.AnimatedMarker() ]) return reader
def _solve_dd(p_e_0, p_h_0, q0, n_steps, dt, geom_descr, pot_w_descr, pot_descr, temp, diffusion, t_e_trapping, t_h_trapping, t_e_t1, t_h_t1, t_r, save_frac): p_e, p_h = p_e_0, p_h_0 # Result arrays initialized to NaN n_store = int(n_steps / save_frac) # Steps to store max_step_size = n_steps / n_store * 10 # Different store time step for each e-h pair T = np.full(shape=(n_store, p_e.shape[1]), fill_value=np.nan, dtype=np.float32) # Stored trajectory for each eh pair traj_e = np.full(shape=(n_store, p_e.shape[0], p_e.shape[1]), fill_value=np.nan) traj_h = np.full(shape=(n_store, p_h.shape[0], p_h.shape[1]), fill_value=np.nan) # Stored induced charge for each eh pair I_ind_e = np.zeros(shape=(n_store, p_e.shape[1])) I_ind_h = np.zeros_like(I_ind_e) # Helper array(s) of actual step index and next step index i_step = np.zeros(p_e.shape[1], dtype=np.int) # Result array indeces next_step = np.zeros_like(i_step) # Next time step to store # Summed induced charge/current with every time step I_ind_tot = np.zeros(shape=(n_steps)) # Total induced charge Q_ind_tot_e = np.zeros(shape=(p_e.shape[1])) Q_ind_tot_h = np.zeros(shape=(p_h.shape[1])) def add_diffusion(v_e, v_h): # Calculate absolute thermal velocity v_th_e = silicon.get_thermal_velocity(temperature=temp, is_electron=True) v_th_h = silicon.get_thermal_velocity(temperature=temp, is_electron=False) # Create thermal velocity distribution # From: The Atomistic Simulation of Thermal Diffusion # and Coulomb Drift in Semiconductor Detectors # IEEE VOL. 56, NO. 3, JUNE 2009 v_th_e *= np.sqrt( 2. / 3. * np.log(np.abs(1. / (1. - np.random.uniform(size=v_e.shape[1]))))) v_th_h *= np.sqrt( 2. / 3. * np.log(np.abs(1. / (1. - np.random.uniform(size=v_h.shape[1]))))) # Calculate random direction in x, y # Uniform random number 0 .. 2 Pi eta = np.random.uniform(0., 2. * np.pi, size=v_e.shape[1]) direction_e = np.array([np.cos(eta), np.sin(eta)]) eta = np.random.uniform(0., 2. * np.pi, size=v_h.shape[1]) direction_h = np.array([np.cos(eta), np.sin(eta)]) v_th_e = v_th_e[np.newaxis, :] * direction_e v_th_h = v_th_h[np.newaxis, :] * direction_h v_e += v_th_e v_h += v_th_h return v_e, v_h def cal_step_size(t, Q_ind_tot, dt, dydt, q_max, i_step, n_store): ''' Calculates the step size from the actual data It is assumed that the actual slope stays constant. The remaining time distance is calculated and the step size is adjusted to fit the remaining steps. ''' step_size = np.zeros_like(q_max) # All steps are used, mark as done sel_done = n_store - i_step - 1 <= 0 # All storage spaces used up if step_size[~sel_done].size == 0: return step_size.astype(np.int) # Set next step to NaN if storing is done step_size[sel_done] = np.NaN # Calculate remaining x distance to cover try: # Case: increasing function (assume value = q_max at tmax) t_max_exp = ((q_max - Q_ind_tot) / dydt + t)[~sel_done] # Case: decreasing function (assume value = 0 at tmax) t_max_exp[dydt[~sel_done] < 0] = ( (-q_max - Q_ind_tot) / dydt + t)[~sel_done][dydt[~sel_done] < 0] # Correct expected time larger than simulation time t_max_exp[t_max_exp > n_steps * dt] = n_steps * dt # Remaining time to cover t_left = t_max_exp - t except (IndexError, ValueError): logging.error('q_max.shape %s', str(q_max.shape)) logging.error('sel_done.shape %s', str(sel_done.shape)) logging.error('Q_ind_tot.shape %s', str(Q_ind_tot.shape)) logging.error('dydt.shape %s', str(dydt.shape)) logging.error('t_max_exp.shape %s', str(t_max_exp.shape)) logging.error('dydt[~sel_done].shape %s', str(dydt[~sel_done].shape)) logging.error('(dydt[~sel_done] < 0).shape %s', str((dydt[~sel_done] < 0).shape)) logging.error('(-q_max - Q_ind_tot) / dydt + t).shape %s', str(((-q_max - Q_ind_tot) / dydt + t).shape)) logging.error( '(-q_max - Q_ind_tot) / dydt + t)[~sel_done].shape %s', str(((-q_max - Q_ind_tot) / dydt + t)[~sel_done].shape)) raise # Calculate the step size step_size[~sel_done] = t_left / dt / (n_store - i_step[~sel_done] - 1) # Limit step size to max_step_size # Needed for slope direction changing functions sel_lim_max = step_size[~sel_done] > max_step_size step_size[~sel_done][sel_lim_max] = max_step_size # Minimum step size = 1 step_size[np.logical_and(~sel_done, step_size < 1.)] = 1 step_size = step_size.astype(np.int) return step_size def store_if_needed(step, next_step, Q_ind_tot_e, Q_ind_tot_h, dt, dQ_e, dQ_h, T, I_ind_e, I_ind_h, p_e, p_h, q_max, i_step, n_store, sel_e, sel_h): ''' Checks if charge carriers value needs to be stored and returns next storage time step ''' t = dt * step # Actual time step # Select charges that need storing for full array (1 entry per e-h) store = step == next_step # Select charges that need storing and are not fully propagated yet # for full array (1 entry per e-h) store_e = np.logical_and(store, sel_e) store_h = np.logical_and(store, sel_h) # Select charges that need storing for reduced array (e-h that need # propagating s_e = store_e[sel_e] s_h = store_h[sel_h] # All carrier stored if not np.any(store): return # All carriers needing storing are fully drifted if not np.any(s_e) and not np.any(s_h): return # Set data of actual time step T[i_step[store], store] = t # Calculate induced charge for integrated time steps T DT_e = T[i_step[store_e], store_e] - T[i_step[store_e] - 1, store_e] DT_h = T[i_step[store_h], store_h] - T[i_step[store_h] - 1, store_h] DT_e[np.isnan(DT_e)] = dt # First storing DT_h[np.isnan(DT_h)] = dt # First storing I_ind_e[i_step[store_e], store_e] = dQ_e_step[store_e] / DT_e I_ind_h[i_step[store_h], store_h] = dQ_h_step[store_h] / DT_h # Store data # I_ind_e[i_step[store_e], store_e] = dQ_e[s_e] / dt # I_ind_h[i_step[store_h], store_h] = dQ_h[s_h] / dt traj_e[i_step[store_e], :, store_e] = p_e[:, store_e].T traj_h[i_step[store_h], :, store_h] = p_h[:, store_h].T # if np.max(i_step) == 145: # plt.plot(T[:, 0], I_ind_e[:, 0], '.') # plt.plot(T[:, 0], I_ind_h[:, 0], '.') # plt.show() # Calculate step size as the minimum of the e and h step size d_step = np.zeros_like(next_step) d_step_e = np.zeros_like(d_step) d_step_h = np.zeros_like(d_step) if np.any(store_e): d_step_e[store_e] = cal_step_size(t, Q_ind_tot=Q_ind_tot_e[store_e], dt=dt, dydt=I_ind_e[i_step[store_e], store_e], q_max=q_max[store_e], i_step=i_step[store_e], n_store=n_store) d_step[store_e] = d_step_e[store_e] if np.any(store_h): d_step_h[store_h] = cal_step_size(t, Q_ind_tot=Q_ind_tot_h[store_h], dt=dt, dydt=I_ind_h[i_step[store_h], store_h], q_max=q_max[store_h], i_step=i_step[store_h], n_store=n_store) d_step[store_h] = d_step_h[store_h] sel = np.logical_and(store_e, store_h) d_step[sel] = np.minimum(d_step_e[sel], d_step_h[sel]) next_step += d_step # Increase storage hists indeces i_step[store] += 1 i_step[i_step >= T.shape[0]] = T.shape[0] - 1 # Reset tmp. variable dQ_e_step[store_e] = 0. dQ_h_step[store_h] = 0. # Tmp. variables to store the total induced charge per save step # Otherwise the resolution of induced current calculation # is reduced dQ_e_step = np.zeros(shape=p_e.shape[1]) dQ_h_step = np.zeros(shape=p_h.shape[1]) progress_bar = progressbar.ProgressBar(widgets=[ '', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA() ], maxval=n_steps, term_width=80) progress_bar.start() sel_e, _ = _in_boundary(geom_descr, pot_descr=pot_descr, x=p_e[0, :], y=p_e[1, :], sel=np.ones(p_e.shape[1], dtype=np.bool)) sel_h, _ = _in_boundary(geom_descr, pot_descr=pot_descr, x=p_h[0, :], y=p_h[1, :], sel=np.ones(p_h.shape[1], dtype=np.bool)) for step in range(n_steps): # Check if all particles out of boundary if not np.any(sel_h) and not np.any(sel_e): break # Stop loop to safe time # Electric field in V/cm E_e = pot_descr.get_field(p_e[0, sel_e], p_e[1, sel_e]) * 1e4 E_h = pot_descr.get_field(p_h[0, sel_h], p_h[1, sel_h]) * 1e4 # Mobility in cm2 / Vs mu_e = silicon.get_mobility(np.sqrt(E_e[0]**2 + E_e[1]**2), temperature=temp, is_electron=True) mu_h = silicon.get_mobility(np.sqrt(E_h[0]**2 + E_h[1]**2), temperature=temp, is_electron=False) # Drift velocity in cm / s v_e, v_h = -E_e * mu_e, E_h * mu_h if diffusion: v_e, v_h = add_diffusion(v_e, v_h) # Calculate induced current # Only if electrons are still drifting if np.any(sel_e): # Weighting field in V/um W_e = pot_w_descr.get_field(p_e[0, sel_e], p_e[1, sel_e]) # Induced charge in C/s, Q = E_w * v * q * dt dQ_e = (W_e[0] * v_e[0] + W_e[1] * v_e[1]) * \ - q0[sel_e] * dt * 1e-5 # Reduce induced charge due to trapping if t_e_trapping: t_e = t_e_trapping + t_e_t1 * np.sqrt(E_e[0]**2 + E_e[1]**2) * 1e-4 dQ_e *= np.exp(-dt * step / t_e) if t_r: dQ_e *= np.exp(-dt * step / (t_r / 2.2)) dQ_e_step[sel_e] += dQ_e Q_ind_tot_e[sel_e] += dQ_e I_ind_tot[step] += dQ_e.sum() / dt if np.any(sel_h): # Only if holes are still drifting # Weighting field in V/um W_h = pot_w_descr.get_field(p_h[0, sel_h], p_h[1, sel_h]) # Induced charge in C/s, Q = E_w * v * q * dt dQ_h = (W_h[0] * v_h[0] + W_h[1] * v_h[1]) * \ q0[sel_h] * dt * 1e-5 # Reduce induced charge due to trapping if t_h_trapping: t_h = t_h_trapping + t_h_t1 * np.sqrt(E_h[0]**2 + E_h[1]**2) * 1e-4 dQ_h *= np.exp(-dt * step / t_h) if t_r: dQ_h *= np.exp(-dt * step / (t_r / 2.2)) dQ_h_step[sel_h] += dQ_h Q_ind_tot_h[sel_h] += dQ_h I_ind_tot[step] += dQ_h.sum() / dt # Store store_if_needed(step, next_step, Q_ind_tot_e=Q_ind_tot_e, Q_ind_tot_h=Q_ind_tot_h, dt=dt, dQ_e=dQ_e, dQ_h=dQ_h, T=T, I_ind_e=I_ind_e, I_ind_h=I_ind_h, p_e=p_e, p_h=p_h, q_max=q0, i_step=i_step, n_store=n_store, sel_e=sel_e, sel_h=sel_h) # Position change in um d_p_e, d_p_h = v_e * dt * 1e-5, v_h * dt * 1e-5 # Update position p_e[:, sel_e] = p_e[:, sel_e] + d_p_e p_h[:, sel_h] = p_h[:, sel_h] + d_p_h # Correct boundaries (e.g. leaving sensor due to diffusion) _correct_boundary(geom_descr, pot_descr, x=p_e[0, :], y=p_e[1, :], sel=sel_e, is_electron=True) _correct_boundary(geom_descr, pot_descr, x=p_h[0, :], y=p_h[1, :], sel=sel_h, is_electron=False) # Check boundaries and update selection sel_e, new_e = _in_boundary(geom_descr, pot_descr=pot_descr, x=p_e[0, :], y=p_e[1, :], sel=sel_e) sel_h, new_h = _in_boundary(geom_descr, pot_descr=pot_descr, x=p_h[0, :], y=p_h[1, :], sel=sel_h) # Force a storing step at for e-h pairs where one is finished next_step[new_e] = step + 1 next_step[new_h] = step + 1 p_e[:, ~sel_e] = np.nan p_h[:, ~sel_h] = np.nan progress_bar.update(step) progress_bar.finish() return traj_e, traj_h, I_ind_e, I_ind_h, T, I_ind_tot, Q_ind_tot_e, Q_ind_tot_h
def create_hitor_calibration(output_filename, plot_pixel_calibrations=False): '''Generating HitOr calibration file (_calibration.h5) from raw data file and plotting of calibration data. Parameters ---------- output_filename : string Input raw data file name. plot_pixel_calibrations : bool, iterable If True, genearating additional pixel calibration plots. If list of column and row tuples (from 1 to 80 / 336), print selected pixels. Returns ------- nothing ''' logging.info('Analyze HitOR calibration data and plot results of %s', output_filename) with AnalyzeRawData(raw_data_file=output_filename, create_pdf=True ) as analyze_raw_data: # Interpret the raw data file analyze_raw_data.create_occupancy_hist = False # too many scan parameters to do in ram histogramming analyze_raw_data.create_hit_table = True analyze_raw_data.create_tdc_hist = True analyze_raw_data.align_at_tdc = True # align events at TDC words, first word of event has to be a tdc word analyze_raw_data.interpret_word_table() analyze_raw_data.interpreter.print_summary() analyze_raw_data.plot_histograms() n_injections = analyze_raw_data.n_injections # use later with tb.open_file( analyze_raw_data._analyzed_data_file, 'r' ) as in_file_h5: # Get scan parameters from interpreted file meta_data = in_file_h5.root.meta_data[:] scan_parameters_dict = get_scan_parameter(meta_data) inner_loop_parameter_values = scan_parameters_dict[next( reversed(scan_parameters_dict) )] # inner loop parameter name is unknown scan_parameter_names = scan_parameters_dict.keys() # col_row_combinations = get_unique_scan_parameter_combinations(in_file_h5.root.meta_data[:], scan_parameters=('column', 'row'), scan_parameter_columns_only=True) meta_data_table_at_scan_parameter = get_unique_scan_parameter_combinations( meta_data, scan_parameters=scan_parameter_names) scan_parameter_values = get_scan_parameters_table_from_meta_data( meta_data_table_at_scan_parameter, scan_parameter_names) event_number_ranges = get_ranges_from_array( meta_data_table_at_scan_parameter['event_number']) event_ranges_per_parameter = np.column_stack( (scan_parameter_values, event_number_ranges)) hits = in_file_h5.root.Hits[:] event_numbers = hits['event_number'].copy( ) # create contigous array, otherwise np.searchsorted too slow, http://stackoverflow.com/questions/15139299/performance-of-numpy-searchsorted-is-poor-on-structured-arrays output_filename = os.path.splitext(output_filename)[0] with tb.open_file(output_filename + "_calibration.h5", mode="w") as calibration_data_file: logging.info('Create calibration') calibration_data = np.full( shape=(80, 336, len(inner_loop_parameter_values), 4), fill_value=np.nan, dtype='f4' ) # result of the calibration is a histogram with col_index, row_index, plsrDAC value, mean discrete tot, rms discrete tot, mean tot from TDC, rms tot from TDC progress_bar = progressbar.ProgressBar( widgets=[ '', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA() ], maxval=len(event_ranges_per_parameter), term_width=80) progress_bar.start() for index, ( actual_scan_parameter_values, event_start, event_stop) in enumerate(event_ranges_per_parameter): if event_stop is None: # happens for the last chunk event_stop = hits[-1]['event_number'] + 1 array_index = np.searchsorted( event_numbers, np.array([event_start, event_stop])) actual_hits = hits[array_index[0]:array_index[1]] for item_index, item in enumerate(scan_parameter_names): if item == "column": actual_col = actual_scan_parameter_values[ item_index] elif item == "row": actual_row = actual_scan_parameter_values[ item_index] elif item == "PlsrDAC": plser_dac = actual_scan_parameter_values[ item_index] else: raise ValueError("Unknown scan parameter %s" % item) # Only pixel of actual column/row should be in the actual data chunk but since SRAM is not cleared for each scan step due to speed reasons and there might be noisy pixels this is not always the case n_wrong_pixel = np.count_nonzero( np.logical_or(actual_hits['column'] != actual_col, actual_hits['row'] != actual_row)) if n_wrong_pixel != 0: logging.warning( '%d hit(s) from other pixels for scan parameters %s', n_wrong_pixel, ', '.join([ '%s=%s' % (name, value) for (name, value ) in zip(scan_parameter_names, actual_scan_parameter_values) ])) actual_hits = actual_hits[np.logical_and( actual_hits['column'] == actual_col, actual_hits['row'] == actual_row)] # Only take data from selected pixel actual_tdc_hits = actual_hits[ (actual_hits['event_status'] & 0b0000111110011100) == 0b0000000100000000] # only take hits from good events (one TDC word only, no error) actual_tot_hits = actual_hits[ (actual_hits['event_status'] & 0b0000100010011100) == 0b0000000000000000] # only take hits from good events for tot tot, tdc = actual_tot_hits['tot'], actual_tdc_hits['TDC'] if tdc.shape[0] < n_injections: logging.info( '%d of %d expected TDC hits for scan parameters %s', tdc.shape[0], n_injections, ', '.join([ '%s=%s' % (name, value) for (name, value ) in zip(scan_parameter_names, actual_scan_parameter_values) ])) if tot.shape[0] < n_injections: logging.info( '%d of %d expected hits for scan parameters %s', tot.shape[0], n_injections, ', '.join([ '%s=%s' % (name, value) for (name, value ) in zip(scan_parameter_names, actual_scan_parameter_values) ])) inner_loop_scan_parameter_index = np.where( plser_dac == inner_loop_parameter_values )[0][ 0] # translate the scan parameter value to an index for the result histogram # numpy mean and std return nan if array is empty calibration_data[actual_col - 1, actual_row - 1, inner_loop_scan_parameter_index, 0] = np.mean(tot) calibration_data[actual_col - 1, actual_row - 1, inner_loop_scan_parameter_index, 1] = np.mean(tdc) calibration_data[actual_col - 1, actual_row - 1, inner_loop_scan_parameter_index, 2] = np.std(tot) calibration_data[actual_col - 1, actual_row - 1, inner_loop_scan_parameter_index, 3] = np.std(tdc) progress_bar.update(index) progress_bar.finish() calibration_data_out = calibration_data_file.create_carray( calibration_data_file.root, name='HitOrCalibration', title='Hit OR calibration data', atom=tb.Atom.from_dtype(calibration_data.dtype), shape=calibration_data.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False)) calibration_data_out[:] = calibration_data calibration_data_out.attrs.dimensions = scan_parameter_names calibration_data_out.attrs.scan_parameter_values = inner_loop_parameter_values calibration_data_out.flush() # with PdfPages(output_filename + "_calibration.pdf") as output_pdf: plot_scurves(calibration_data[:, :, :, 0], inner_loop_parameter_values, "ToT calibration", "ToT", 15, "Charge [PlsrDAC]", filename=analyze_raw_data.output_pdf) plot_scurves(calibration_data[:, :, :, 1], inner_loop_parameter_values, "TDC calibration", "TDC [ns]", None, "Charge [PlsrDAC]", filename=analyze_raw_data.output_pdf) tot_mean_all_pix = np.nanmean(calibration_data[:, :, :, 0], axis=(0, 1)) tot_error_all_pix = np.nanstd(calibration_data[:, :, :, 0], axis=(0, 1)) tdc_mean_all_pix = np.nanmean(calibration_data[:, :, :, 1], axis=(0, 1)) tdc_error_all_pix = np.nanstd(calibration_data[:, :, :, 1], axis=(0, 1)) plot_tot_tdc_calibration( scan_parameters=inner_loop_parameter_values, tot_mean=tot_mean_all_pix, tot_error=tot_error_all_pix, tdc_mean=tdc_mean_all_pix, tdc_error=tdc_error_all_pix, filename=analyze_raw_data.output_pdf, title="Mean charge calibration of %d pixel(s)" % np.count_nonzero(~np.all( np.isnan(calibration_data[:, :, :, 0]), axis=2))) # plotting individual pixels if plot_pixel_calibrations is True: # selecting pixels with non-nan entries col_row_non_nan = np.nonzero(~np.all( np.isnan(calibration_data[:, :, :, 0]), axis=2)) plot_pixel_calibrations = np.dstack(col_row_non_nan)[0] elif plot_pixel_calibrations is False: plot_pixel_calibrations = np.array([], dtype=np.int) else: # assuming list of column / row tuples plot_pixel_calibrations = np.array( plot_pixel_calibrations) - 1 # generate index array pixel_indices = np.arange(plot_pixel_calibrations.shape[0]) plot_n_pixels = 10 # number of pixels at the beginning, center and end of the array np.random.seed(0) # select random pixels if pixel_indices.size - 2 * plot_n_pixels >= 0: random_pixel_indices = np.sort( np.random.choice( pixel_indices[plot_n_pixels:-plot_n_pixels], min(plot_n_pixels, pixel_indices.size - 2 * plot_n_pixels), replace=False)) else: random_pixel_indices = np.array([], dtype=np.int) selected_pixel_indices = np.unique( np.hstack([ pixel_indices[:plot_n_pixels], random_pixel_indices, pixel_indices[-plot_n_pixels:] ])) # plotting individual pixels for (column, row) in plot_pixel_calibrations[selected_pixel_indices]: logging.info( "Plotting charge calibration for pixel column " + str(column + 1) + " / row " + str(row + 1)) tot_mean_single_pix = calibration_data[column, row, :, 0] tot_std_single_pix = calibration_data[column, row, :, 2] tdc_mean_single_pix = calibration_data[column, row, :, 1] tdc_std_single_pix = calibration_data[column, row, :, 3] plot_tot_tdc_calibration( scan_parameters=inner_loop_parameter_values, tot_mean=tot_mean_single_pix, tot_error=tot_std_single_pix, tdc_mean=tdc_mean_single_pix, tdc_error=tdc_std_single_pix, filename=analyze_raw_data.output_pdf, title="Charge calibration for pixel column " + str(column + 1) + " / row " + str(row + 1))
def update(self, pbar): try: return str(sets[pbar.currval-1]) except IndexError: return '' widgets = ['Benchmark: ', pbar.Percentage(), ' ', pbar.Counter(), '/', str(n), ' ', Counter(), ' ', pbar.Bar(marker='-'), ' ', pbar.AdaptiveETA(), ' ', ] pbar = pbar.ProgressBar(widgets=widgets, maxval=n).start() # go johnny go, go! for i, it in enumerate(sets): size, storage, entropy, codec, level = it if size == 'small': number = 10 repeat = 10 elif size == 'mid': number = 5 repeat = 5 elif size == 'large':
def analyze(self): logging.info('Analysing the PlsrDAC waveforms') with tb.open_file(self.output_filename + '.h5', 'r') as in_file_h5: data = in_file_h5.root.PlsrDACwaveforms[:] times = np.array(in_file_h5.root.PlsrDACwaveforms._v_attrs.times) scan_parameter_values = in_file_h5.root.PlsrDACwaveforms._v_attrs.scan_parameter_values trigger_levels = in_file_h5.root.PlsrDACwaveforms._v_attrs.trigger_levels fit_range = ast.literal_eval( in_file_h5.root.configuration.run_conf[:][np.where( in_file_h5.root.configuration.run_conf[:]['name'] == 'fit_range')]['value'][0]) fit_range_step = ast.literal_eval( in_file_h5.root.configuration.run_conf[:][np.where( in_file_h5.root.configuration.run_conf[:]['name'] == 'fit_range_step')]['value'][0]) progress_bar = progressbar.ProgressBar(widgets=[ '', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA() ], maxval=data.shape[0], term_width=80) with tb.open_file(self.output_filename + '_interpreted.h5', 'w') as out_file_h5: description = [('PlsrDAC', np.uint32), ('voltage_step', np.float) ] # output data table description data_array = np.zeros((data.shape[0], ), dtype=description) data_table = out_file_h5.create_table( out_file_h5.root, name='plsr_dac_data', description=np.zeros((1, ), dtype=description).dtype, title= 'Voltage steps from transient PlsrDAC calibration scan') with PdfPages(self.output_filename + '_interpreted.pdf') as output_pdf: progress_bar.start() for index in range(data.shape[0]): voltages = data[index] trigger_level = trigger_levels[index] plsr_dac = scan_parameter_values[index] if abs(trigger_level) < 0.005: logging.warning( 'The trigger threshold for PlsrDAC %d is with %d mV too low. Thus this setting is omitted in the analysis!', plsr_dac, trigger_level * 1000.) data_array['voltage_step'][index] = np.NaN continue step_index = np.where( np.abs(voltages - trigger_level) == np.amin( np.abs(voltages - trigger_level)))[0][0] left_step_fit_range = (step_index + fit_range_step[0][0], step_index + fit_range_step[0][1]) right_step_fit_range = (step_index + fit_range_step[1][0], step_index + fit_range_step[1][1]) # Error handling if selected fit range exeeds limits if left_step_fit_range[0] < 0 or left_step_fit_range[ 1] < 0 or right_step_fit_range[0] >= data.shape[ 1] or right_step_fit_range[1] >= data.shape[ 1] or left_step_fit_range[ 0] >= left_step_fit_range[ 1] or right_step_fit_range[ 0] >= right_step_fit_range[ 1]: logging.warning( 'The step fit limits for PlsrDAC %d are out of bounds. Omit this data!', plsr_dac) data_array['voltage_step'][index] = np.NaN continue times_left_step, voltage_left_step = times[ left_step_fit_range[0]: left_step_fit_range[1]], voltages[ left_step_fit_range[0]:left_step_fit_range[1]] times_right_step, voltage_right_step = times[ right_step_fit_range[0]:right_step_fit_range[ 1]], voltages[right_step_fit_range[0]: right_step_fit_range[1]] median_left_step = np.median(voltage_left_step) median_right_step = np.median(voltage_right_step) data_array['PlsrDAC'][index] = plsr_dac data_array['voltage_step'][ index] = median_left_step - median_right_step # Plot waveform + fit plt.clf() plt.grid() plt.plot(times * 1e9, voltages * 1e3, label='Data') plt.plot(times * 1e9, np.repeat([trigger_level * 1e3], len(times)), '--', label='Trigger (%d mV)' % (trigger_level * 1000)) plt.plot(times_left_step * 1e9, np.repeat(median_left_step * 1e3, times_left_step.shape[0]), '-', linewidth=2, label='Left of step constant fit') plt.plot(times_right_step * 1e9, np.repeat(median_right_step * 1e3, times_right_step.shape[0]), '-', linewidth=2, label='Right of step constant fit') plt.title('PulserDAC %d waveform' % plsr_dac) plt.xlabel('Time [ns]') plt.ylabel('Voltage [mV]') plt.legend(loc=0) output_pdf.savefig() progress_bar.update(index) data_table.append(data_array[np.isfinite( data_array['voltage_step'])]) # store valid data # Plot, fit and store linear PlsrDAC transfer function x, y = data_array[np.isfinite( data_array['voltage_step'])]['PlsrDAC'], data_array[ np.isfinite( data_array['voltage_step'])]['voltage_step'] fit = polyfit( x[np.logical_and(x >= fit_range[0], x <= fit_range[1])], y[np.logical_and(x >= fit_range[0], x <= fit_range[1])], 1) fit_fn = poly1d(fit) plt.clf() plt.plot(x, y, '.-', label='data') plt.plot(x, fit_fn(x), '--k', label=str(fit_fn)) plt.title('PlsrDAC calibration') plt.xlabel('PlsrDAC') plt.ylabel('Voltage step [V]') plt.grid(True) plt.legend(loc=0) output_pdf.savefig() # Store result in file self.register.calibration_parameters[ 'Vcal_Coeff_0'] = fit[1] * 1000. # store in mV self.register.calibration_parameters[ 'Vcal_Coeff_1'] = fit[0] * 1000. # store in mV/DAC progress_bar.finish()
def histogram_tdc_hits(input_file_hits, hit_selection_conditions, event_status_select_mask, event_status_condition, calibration_file=None, correct_calibration=None, max_tdc=1000, ignore_disabled_regions=True, n_bins=200, plot_data=True): for condition in hit_selection_conditions: logging.info('Histogram TDC hits with %s', condition) def get_charge( max_tdc, tdc_calibration_values, tdc_pixel_calibration): # return the charge from calibration charge_calibration = np.zeros(shape=(80, 336, max_tdc)) for column in range(80): for row in range(336): actual_pixel_calibration = tdc_pixel_calibration[column, row, :] # Only take pixels with at least 3 valid calibration points if np.count_nonzero( actual_pixel_calibration != 0 ) > 2 and np.count_nonzero( np.isfinite(actual_pixel_calibration)) > 2: selected_measurements = np.isfinite( actual_pixel_calibration ) # Select valid calibration steps selected_actual_pixel_calibration = actual_pixel_calibration[ selected_measurements] selected_tdc_calibration_values = tdc_calibration_values[ selected_measurements] interpolation = interp1d( x=selected_actual_pixel_calibration, y=selected_tdc_calibration_values, kind='slinear', bounds_error=False, fill_value=0) charge_calibration[column, row, :] = interpolation( np.arange(max_tdc)) return charge_calibration def plot_tdc_tot_correlation(data, condition, output_pdf): logging.info('Plot correlation histogram for %s', condition) data = np.ma.array(data, mask=(data <= 0)) if np.ma.any(data > 0): fig = Figure() FigureCanvas(fig) ax = fig.add_subplot(111) cmap = cm.get_cmap('jet', 200) cmap.set_bad('w') ax.set_title('Correlation with %s' % condition) norm = colors.LogNorm() z_max = data.max(fill_value=0) ax.set_xlabel('TDC') ax.set_ylabel('TOT') im = ax.imshow(data, cmap=cmap, norm=norm, aspect='auto', interpolation='nearest') # , norm=norm) divider = make_axes_locatable(ax) ax.invert_yaxis() cax = divider.append_axes("right", size="5%", pad=0.1) fig.colorbar(im, cax=cax, ticks=np.linspace(start=0, stop=z_max, num=9, endpoint=True)) output_pdf.savefig(fig) else: logging.warning('No data for correlation plotting for %s', condition) def plot_hits_per_condition(output_pdf): logging.info( 'Plot hits selection efficiency histogram for %d conditions', len(hit_selection_conditions) + 2) labels = ['All Hits', 'Hits of\ngood events'] for condition in hit_selection_conditions: condition = re.sub('[&]', '\n', condition) condition = re.sub('[()]', '', condition) labels.append(condition) fig = Figure() FigureCanvas(fig) ax = fig.add_subplot(111) ax.bar(range(len(n_hits_per_condition)), n_hits_per_condition, align='center') ax.set_xticks(range(len(n_hits_per_condition)), labels) ax.tick_params(axis='x', labelsize=8) ax.set_title('Number of hits for different cuts') ax.set_yscale('log') ax.set_ylabel('#') ax.grid(True) for x, y in zip(np.arange(len(n_hits_per_condition)), n_hits_per_condition): ax.annotate('%d' % (float(y) / float(n_hits_per_condition[0]) * 100.0) + r'%', xy=(x, y / 2.0), xycoords='data', color='grey', size=15) output_pdf.savefig(fig) def plot_corrected_tdc_hist(x, y, title, output_pdf, point_style='-'): logging.info('Plot TDC hist with TDC calibration') fig = Figure() FigureCanvas(fig) ax = fig.add_subplot(111) y /= np.amax(y) if y.shape[0] > 0 else y ax.plot(x, y, point_style) ax.set_title(title, size=10) ax.set_xlabel('Charge [PlsrDAC]') ax.set_ylabel('Count [a.u.]') ax.grid(True) output_pdf.savefig(fig) def get_calibration_correction( tdc_calibration, tdc_calibration_values, filename_new_calibration ): # correct the TDC calibration with the TDC calib in filename_new_calibration by shifting the means with tb.open_file(filename_new_calibration, 'r') as in_file_2: with PdfPages( os.path.splitext(filename_new_calibration)[0] + '.pdf') as output_pdf: charge_calibration_1, charge_calibration_2 = tdc_calibration, in_file_2.root.HitOrCalibration[:, :, :, 1] plsr_dacs = tdc_calibration_values if not np.all(plsr_dacs == in_file_2.root.HitOrCalibration. _v_attrs.scan_parameter_values): raise NotImplementedError( 'The check calibration file has to have the same PlsrDAC values' ) # Valid pixel have a calibration in the new and the old calibration valid_pixel = np.where( ~np.all((charge_calibration_1 == 0), axis=2) & ~np.all(np.isnan(charge_calibration_1), axis=2) & ~np.all((charge_calibration_2 == 0), axis=2) & ~np.all(np.isnan(charge_calibration_2), axis=2)) mean_charge_calibration = np.nanmean( charge_calibration_2[valid_pixel], axis=0) offset_mean = np.nanmean((charge_calibration_2[valid_pixel] - charge_calibration_1[valid_pixel]), axis=0) dPlsrDAC_dTDC = analysis_utils.smooth_differentiation( plsr_dacs, mean_charge_calibration, order=3, smoothness=0, derivation=1) fig = Figure() FigureCanvas(fig) ax = fig.add_subplot(111) ax.plot(plsr_dacs, offset_mean / dPlsrDAC_dTDC, '.-', label='PlsrDAC') ax.plot(plsr_dacs, offset_mean, '.-', label='TDC') ax.grid(True) ax.set_xlabel('PlsrDAC') ax.set_ylabel('Mean calibration offset') ax.legend(loc=0) ax.set_title( 'Mean offset between TDC calibration data, new - old ') output_pdf.savefig(fig) return offset_mean def delete_disabled_regions(hits, enable_mask): n_hits = hits.shape[0] # Tread no hits case if n_hits == 0: return hits # Column, row array with True for disabled pixels disabled_mask = ~enable_mask.astype(np.bool).T.copy() disabled_region = disabled_mask.copy() n_disabled_pixels = np.count_nonzero(disabled_region) # Extend disabled pixel mask by the neighbouring pixels neighbour_pixels = [(-1, 0), (1, 0), (0, -1), (0, 1)] # Disable direct neighbouring pixels for neighbour_pixel in neighbour_pixels: disabled_region = np.logical_or( disabled_region, shift(disabled_mask, shift=neighbour_pixel, cval=0)) logging.info( 'Masking %d additional pixel neighbouring %d disabled pixels', np.count_nonzero(disabled_region) - n_disabled_pixels, n_disabled_pixels) # Make 1D selection array with disabled pixels disabled_pixels = np.where(disabled_region) disabled_pixels_1d = ( disabled_pixels[0] + 1) * disabled_region.shape[1] + ( disabled_pixels[1] + 1 ) # + 1 because pixel index 0,0 has column/row = 1 hits_1d = hits['column'].astype( np.uint32) * disabled_region.shape[1] + hits[ 'row'] # change dtype to fit new number hits = hits[np.in1d(hits_1d, disabled_pixels_1d, invert=True)] logging.info('Lost %d hits (%d percent) due to disabling neighbours', n_hits - hits.shape[0], (1. - float(hits.shape[0]) / n_hits) * 100) return hits # Create data with tb.open_file(input_file_hits, mode="r") as in_hit_file_h5: cluster_hit_table = in_hit_file_h5.root.ClusterHits try: enabled_pixels = in_hit_file_h5.root.ClusterHits._v_attrs.enabled_pixels[:] except AttributeError: # Old and simulate data do not have this info logging.warning( 'No enabled pixel mask found in data! Assume all pixels are enabled.' ) enabled_pixels = np.ones(shape=(336, 80)) # Result hists, initialized per condition pixel_tdc_hists_per_condition = [ np.zeros(shape=(80, 336, max_tdc), dtype=np.uint16) for _ in hit_selection_conditions ] if hit_selection_conditions else [] pixel_tdc_timestamp_hists_per_condition = [ np.zeros(shape=(80, 336, 256), dtype=np.uint16) for _ in hit_selection_conditions ] if hit_selection_conditions else [] mean_pixel_tdc_hists_per_condition = [ np.zeros(shape=(80, 336), dtype=np.uint16) for _ in hit_selection_conditions ] if hit_selection_conditions else [] mean_pixel_tdc_timestamp_hists_per_condition = [ np.zeros(shape=(80, 336), dtype=np.uint16) for _ in hit_selection_conditions ] if hit_selection_conditions else [] tdc_hists_per_condition = [ np.zeros(shape=(max_tdc), dtype=np.uint16) for _ in hit_selection_conditions ] if hit_selection_conditions else [] tdc_corr_hists_per_condition = [ np.zeros(shape=(max_tdc, 16), dtype=np.uint32) for _ in hit_selection_conditions ] if hit_selection_conditions else [] n_hits_per_condition = [ 0 for _ in range(len(hit_selection_conditions) + 2) ] # condition 1, 2 are all hits, hits of goode events logging.info( 'Select hits and create TDC histograms for %d cut conditions', len(hit_selection_conditions)) progress_bar = progressbar.ProgressBar( widgets=[ '', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA() ], maxval=cluster_hit_table.shape[0], term_width=80) progress_bar.start() for cluster_hits, _ in analysis_utils.data_aligned_at_events( cluster_hit_table, chunk_size=10000000): n_hits_per_condition[0] += cluster_hits.shape[0] selected_events_cluster_hits = cluster_hits[np.logical_and( cluster_hits['TDC'] < max_tdc, (cluster_hits['event_status'] & event_status_select_mask) == event_status_condition)] n_hits_per_condition[1] += selected_events_cluster_hits.shape[0] for index, condition in enumerate(hit_selection_conditions): selected_cluster_hits = analysis_utils.select_hits( selected_events_cluster_hits, condition) if ignore_disabled_regions: selected_cluster_hits = delete_disabled_regions( hits=selected_cluster_hits, enable_mask=enabled_pixels) n_hits_per_condition[2 + index] += selected_cluster_hits.shape[0] column, row, tdc = selected_cluster_hits[ 'column'] - 1, selected_cluster_hits[ 'row'] - 1, selected_cluster_hits['TDC'] pixel_tdc_hists_per_condition[ index] += fast_analysis_utils.hist_3d_index( column, row, tdc, shape=(80, 336, max_tdc)) mean_pixel_tdc_hists_per_condition[index] = np.average( pixel_tdc_hists_per_condition[index], axis=2, weights=range(0, max_tdc)) * np.sum(np.arange( 0, max_tdc)) / pixel_tdc_hists_per_condition[index].sum( axis=2) tdc_timestamp = selected_cluster_hits['TDC_time_stamp'] pixel_tdc_timestamp_hists_per_condition[ index] += fast_analysis_utils.hist_3d_index(column, row, tdc_timestamp, shape=(80, 336, 256)) mean_pixel_tdc_timestamp_hists_per_condition[ index] = np.average( pixel_tdc_timestamp_hists_per_condition[index], axis=2, weights=range(0, 256)) * np.sum(np.arange( 0, 256)) / pixel_tdc_timestamp_hists_per_condition[ index].sum(axis=2) tdc_hists_per_condition[index] = pixel_tdc_hists_per_condition[ index].sum( axis=(0, 1), dtype=np.uint32 ) # fix dtype, sum will otherwise increase precision tdc_corr_hists_per_condition[ index] += fast_analysis_utils.hist_2d_index( tdc, selected_cluster_hits['tot'], shape=(max_tdc, 16)) progress_bar.update(n_hits_per_condition[0]) progress_bar.finish() # Take TDC calibration if available and calculate charge for each TDC value and pixel if calibration_file is not None: with tb.open_file(calibration_file, mode="r") as in_file_calibration_h5: tdc_calibration = in_file_calibration_h5.root.HitOrCalibration[:, :, :, 1] tdc_calibration_values = in_file_calibration_h5.root.HitOrCalibration.attrs.scan_parameter_values[:] if correct_calibration is not None: tdc_calibration += get_calibration_correction( tdc_calibration, tdc_calibration_values, correct_calibration) charge_calibration = get_charge(max_tdc, tdc_calibration_values, tdc_calibration) else: charge_calibration = None # Store data of result histograms with tb.open_file(os.path.splitext(input_file_hits)[0] + '_tdc_hists.h5', mode="w") as out_file_h5: for index, condition in enumerate(hit_selection_conditions): pixel_tdc_hist_result = np.swapaxes( pixel_tdc_hists_per_condition[index], 0, 1) pixel_tdc_timestamp_hist_result = np.swapaxes( pixel_tdc_timestamp_hists_per_condition[index], 0, 1) mean_pixel_tdc_hist_result = np.swapaxes( mean_pixel_tdc_hists_per_condition[index], 0, 1) mean_pixel_tdc_timestamp_hist_result = np.swapaxes( mean_pixel_tdc_timestamp_hists_per_condition[index], 0, 1) tdc_hists_per_condition_result = tdc_hists_per_condition[index] tdc_corr_hist_result = np.swapaxes( tdc_corr_hists_per_condition[index], 0, 1) # Create result hists out_1 = out_file_h5.create_carray( out_file_h5.root, name='HistPixelTdcCondition_%d' % index, title='Hist Pixel Tdc with %s' % condition, atom=tb.Atom.from_dtype(pixel_tdc_hist_result.dtype), shape=pixel_tdc_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False)) out_2 = out_file_h5.create_carray( out_file_h5.root, name='HistPixelTdcTimestampCondition_%d' % index, title='Hist Pixel Tdc Timestamp with %s' % condition, atom=tb.Atom.from_dtype( pixel_tdc_timestamp_hist_result.dtype), shape=pixel_tdc_timestamp_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False)) out_3 = out_file_h5.create_carray( out_file_h5.root, name='HistMeanPixelTdcCondition_%d' % index, title='Hist Mean Pixel Tdc with %s' % condition, atom=tb.Atom.from_dtype(mean_pixel_tdc_hist_result.dtype), shape=mean_pixel_tdc_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False)) out_4 = out_file_h5.create_carray( out_file_h5.root, name='HistMeanPixelTdcTimestampCondition_%d' % index, title='Hist Mean Pixel Tdc Timestamp with %s' % condition, atom=tb.Atom.from_dtype( mean_pixel_tdc_timestamp_hist_result.dtype), shape=mean_pixel_tdc_timestamp_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False)) out_5 = out_file_h5.create_carray( out_file_h5.root, name='HistTdcCondition_%d' % index, title='Hist Tdc with %s' % condition, atom=tb.Atom.from_dtype( tdc_hists_per_condition_result.dtype), shape=tdc_hists_per_condition_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False)) out_6 = out_file_h5.create_carray( out_file_h5.root, name='HistTdcCorrCondition_%d' % index, title='Hist Correlation Tdc/Tot with %s' % condition, atom=tb.Atom.from_dtype(tdc_corr_hist_result.dtype), shape=tdc_corr_hist_result.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False)) # Add result hists information out_1.attrs.dimensions, out_1.attrs.condition, out_1.attrs.tdc_values = 'column, row, TDC value', condition, range( max_tdc) out_2.attrs.dimensions, out_2.attrs.condition, out_2.attrs.tdc_values = 'column, row, TDC time stamp value', condition, range( 256) out_3.attrs.dimensions, out_3.attrs.condition = 'column, row, mean TDC value', condition out_4.attrs.dimensions, out_4.attrs.condition = 'column, row, mean TDC time stamp value', condition out_5.attrs.dimensions, out_5.attrs.condition = 'PlsrDAC', condition out_6.attrs.dimensions, out_6.attrs.condition = 'TDC, TOT', condition out_1[:], out_2[:], out_3[:], out_4[:], out_5[:], out_6[:] = pixel_tdc_hist_result, pixel_tdc_timestamp_hist_result, mean_pixel_tdc_hist_result, mean_pixel_tdc_timestamp_hist_result, tdc_hists_per_condition_result, tdc_corr_hist_result if charge_calibration is not None: # Select only valid pixel for histogramming: they have data and a calibration (that is any charge(TDC) calibration != 0) valid_pixel = np.where( np.logical_and( charge_calibration[:, :, :max_tdc].sum(axis=2) > 0, pixel_tdc_hist_result[:, :, :max_tdc].swapaxes( 0, 1).sum(axis=2) > 0)) # Create charge histogram with mean TDC calibration mean_charge_calibration = charge_calibration[ valid_pixel][:, :max_tdc].mean(axis=0) mean_tdc_hist = pixel_tdc_hist_result.swapaxes( 0, 1)[valid_pixel][:, :max_tdc].mean(axis=0) result_array = np.rec.array(np.column_stack( (mean_charge_calibration, mean_tdc_hist)), dtype=[('charge', float), ('count', float)]) out_7 = out_file_h5.create_table( out_file_h5.root, name='HistMeanTdcCalibratedCondition_%d' % index, description=result_array.dtype, title='Hist Tdc with mean charge calibration and %s' % condition, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False)) out_7.attrs.condition = condition out_7.attrs.n_pixel = valid_pixel[0].shape[0] out_7.attrs.n_hits = pixel_tdc_hist_result.swapaxes( 0, 1)[valid_pixel][:, :max_tdc].sum() out_7.append(result_array) # Create charge histogram with per pixel TDC calibration x, y = charge_calibration[valid_pixel][:, :max_tdc].ravel( ), np.ravel( pixel_tdc_hist_result.swapaxes( 0, 1)[valid_pixel][:, :max_tdc].ravel()) y_hist, x_hist = y[x > 0], x[ x > 0] # remove the hit tdcs without proper calibration plsrDAC(TDC) calibration x, y, yerr = analysis_utils.get_profile_histogram( x_hist, y_hist, n_bins=n_bins) result_array = np.rec.array(np.column_stack((x, y, yerr)), dtype=[('charge', float), ('count', float), ('count_error', float)]) out_8 = out_file_h5.create_table( out_file_h5.root, name='HistTdcCalibratedCondition_%d' % index, description=result_array.dtype, title= 'Hist Tdc with per pixel charge calibration and %s' % condition, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False)) out_8.attrs.condition = condition out_8.attrs.n_pixel = valid_pixel[0].shape[0] out_8.attrs.n_hits = y_hist.sum() out_8.append(result_array) # Plot Data if plot_data: with PdfPages( os.path.splitext(input_file_hits)[0] + '_calibrated_tdc_hists.pdf') as output_pdf: plot_hits_per_condition(output_pdf) with tb.open_file(os.path.splitext(input_file_hits)[0] + '_tdc_hists.h5', mode="r") as in_file_h5: for node in in_file_h5.root: # go through the data and plot them if 'MeanPixel' in node.name: try: plot_three_way( np.ma.masked_invalid(node[:]) * 1.5625, title='Mean TDC delay, hits with\n%s' % node._v_attrs.condition[:80] if 'Timestamp' in node.name else 'Mean TDC, hits with\n%s' % node._v_attrs.condition[:80], filename=output_pdf) except ValueError: logging.warning('Cannot plot TDC delay') elif 'HistTdcCondition' in node.name: hist_1d = node[:] entry_index = np.where(hist_1d != 0) if entry_index[0].shape[0] != 0: max_index = np.amax(entry_index) else: max_index = max_tdc plot_1d_hist( hist_1d[:max_index + 10], title='TDC histogram, hits with\n%s' % node._v_attrs.condition[:80] if 'Timestamp' not in node.name else 'TDC time stamp histogram, hits with\n%s' % node._v_attrs.condition[:80], x_axis_title='TDC' if 'Timestamp' not in node.name else 'TDC time stamp', filename=output_pdf) elif 'HistPixelTdc' in node.name: hist_3d = node[:] entry_index = np.where(hist_3d.sum(axis=(0, 1)) != 0) if entry_index[0].shape[0] != 0: max_index = np.amax(entry_index) else: max_index = max_tdc best_pixel_index = np.where( hist_3d.sum(axis=2) == np.amax(node[:].sum( axis=2))) if best_pixel_index[0].shape[ 0] == 1: # there could be more than one pixel with most hits try: plot_1d_hist( hist_3d[best_pixel_index][0, :max_index], title='TDC histogram of pixel %d, %d\n%s' % (best_pixel_index[1] + 1, best_pixel_index[0] + 1, node._v_attrs.condition[:80]) if 'Timestamp' not in node.name else 'TDC time stamp histogram, hits of pixel %d, %d' % (best_pixel_index[1] + 1, best_pixel_index[0] + 1), x_axis_title='TDC' if 'Timestamp' not in node.name[:80] else 'TDC time stamp', filename=output_pdf) except IndexError: logging.warning( 'Cannot plot pixel TDC histogram') elif 'HistTdcCalibratedCondition' in node.name: plot_corrected_tdc_hist( node[:]['charge'], node[:]['count'], title= 'TDC histogram, %d pixel, per pixel TDC calib.\n%s' % (node._v_attrs.n_pixel, node._v_attrs.condition[:80]), output_pdf=output_pdf) elif 'HistMeanTdcCalibratedCondition' in node.name: plot_corrected_tdc_hist( node[:]['charge'], node[:]['count'], title='TDC histogram, %d pixel, mean TDC calib.\n%s' % (node._v_attrs.n_pixel, node._v_attrs.condition[:80]), output_pdf=output_pdf) elif 'HistTdcCorr' in node.name: plot_tdc_tot_correlation(node[:], node._v_attrs.condition, output_pdf)
def __init__(self, trainer): default_params = { "report_times_every_epoch": None, "report_interval_iters": 100, "record_file": "train.csv", "use_tensorboard": False } self.trainer = trainer default_params = utils.assign_params_dict(default_params, self.trainer.params) if default_params["report_times_every_epoch"] is not None: self.report_interval_iters = max( 1, self.trainer.training_point[2] // default_params["report_times_every_epoch"]) else: self.report_interval_iters = default_params[ "report_interval_iters"] if default_params["use_tensorboard"]: from tensorboardX import SummaryWriter model_name = os.path.basename(self.trainer.params["model_dir"]) time_string = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime(time.time())) self.board_writer = SummaryWriter( "{}/log/{}-{}-tensorboard".format( self.trainer.params["model_dir"], model_name, time_string)) else: self.board_writer = None self.epochs = self.trainer.params["epochs"] self.optimizer = self.trainer.elements["optimizer"] # For optimizer wrapper such as lookahead. if getattr(self.optimizer, "optimizer", None) is not None: self.optimizer = self.optimizer.optimizer self.device = "[{0}]".format( utils.get_device(self.trainer.elements["model"])) self.record_value = [] self.start_write_log = False if default_params["record_file"] != "" and default_params[ "record_file"] is not None: self.record_file = "{0}/log/{1}".format( self.trainer.params["model_dir"], default_params["record_file"]) # The case to recover training if self.trainer.params["start_epoch"] > 0: self.start_write_log = True elif os.path.exists(self.record_file): # Do backup to avoid clearing the loss log when re-running a same launcher. bk_file = "{0}.bk.{1}".format( self.record_file, time.strftime('%Y_%m_%d.%H_%M_%S', time.localtime(time.time()))) shutil.move(self.record_file, bk_file) else: self.record_file = None # A format to show progress # Do not use progressbar.Bar(marker="\x1b[32m█\x1b[39m") and progressbar.SimpleProgress(format='%(value_s)s/%(max_value_s)s') to avoid too long string. widgets = [ progressbar.Percentage(format='%(percentage)3.2f%%'), " | ", "Epoch:", progressbar.Variable('current_epoch', format='{formatted_value}', width=0, precision=0), "/{0}, ".format(self.epochs), "Iter:", progressbar.Variable('current_iter', format='{formatted_value}', width=0, precision=0), "/{0}".format(self.trainer.training_point[2]), " (", progressbar.Timer(format='ELA: %(elapsed)s'), ", ", progressbar.AdaptiveETA(), ")" ] max_value = self.trainer.params[ "epochs"] * self.trainer.training_point[2] self.bar = progressbar.ProgressBar(max_value=max_value, widgets=widgets, redirect_stdout=True) # Use multi-process for update. self.queue = Queue() self.process = Process(target=self._update, daemon=True) self.process.start()
def fit(self, X, B, T, W=None): '''Fits the model. :param X: numpy matrix of shape :math:`k \\cdot n` :param B: numpy vector of shape :math:`n` :param T: numpy vector of shape :math:`n` :param W: (optional) numpy vector of shape :math:`n` ''' if W is None: W = numpy.ones(len(X)) X, B, T, W = (Z if type(Z) == numpy.ndarray else numpy.array(Z) for Z in (X, B, T, W)) keep_indexes = (T > 0) & (B >= 0) & (B <= 1) & (W >= 0) if sum(keep_indexes) < X.shape[0]: n_removed = X.shape[0] - sum(keep_indexes) warnings.warn('Warning! Removed %d/%d entries from inputs where ' 'T <= 0 or B not 0/1 or W < 0' % (n_removed, len(X))) X, B, T, W = (Z[keep_indexes] for Z in (X, B, T, W)) n_features = X.shape[1] # scipy.optimize and emcee forces the the parameters to be a vector: # (log k, log p, log sigma_alpha, log sigma_beta, # a, b, alpha_1...alpha_k, beta_1...beta_k) # Generalized Gamma is a bit sensitive to the starting point! x0 = numpy.zeros(6 + 2 * n_features) x0[0] = +1 if self._fix_k is None else log(self._fix_k) x0[1] = -1 if self._fix_p is None else log(self._fix_p) args = (X, B, T, W, self._fix_k, self._fix_p, self._hierarchical, self._flavor) # Set up progressbar and callback bar = progressbar.ProgressBar(widgets=[ progressbar.Variable('loss', width=15, precision=9), ' ', progressbar.BouncingBar(), ' ', progressbar.Counter(width=6), ' [', progressbar.Timer(), ']' ]) def callback(LL, value_history=[]): value_history.append(LL) bar.update(len(value_history), loss=LL) # Define objective and use automatic differentiation f = lambda x: -generalized_gamma_loss(x, *args, callback=callback) jac = autograd.grad(lambda x: -generalized_gamma_loss(x, *args)) # Find the maximum a posteriori of the distribution res = scipy.optimize.minimize(f, x0, jac=jac, method='SLSQP', options={'maxiter': 9999}) if not res.success: raise Exception('Optimization failed with message: %s' % res.message) result = {'map': res.x} # TODO: should not use fixed k/p as search parameters if self._fix_k: result['map'][0] = log(self._fix_k) if self._fix_p: result['map'][1] = log(self._fix_p) # Make sure we're in a local minimum gradient = jac(result['map']) gradient_norm = numpy.dot(gradient, gradient) if gradient_norm >= 1e-2 * len(X): warnings.warn('Might not have found a local minimum! ' 'Norm of gradient is %f' % gradient_norm) # Let's sample from the posterior to compute uncertainties if self._ci: dim, = res.x.shape n_walkers = 5 * dim sampler = emcee.EnsembleSampler( nwalkers=n_walkers, ndim=dim, log_prob_fn=generalized_gamma_loss, args=args, ) mcmc_initial_noise = 1e-3 p0 = [ result['map'] + mcmc_initial_noise * numpy.random.randn(dim) for i in range(n_walkers) ] n_burnin = 100 n_steps = numpy.ceil(2000. / n_walkers) n_iterations = n_burnin + n_steps bar = progressbar.ProgressBar(max_value=n_iterations, widgets=[ progressbar.Percentage(), ' ', progressbar.Bar(), ' %d walkers [' % n_walkers, progressbar.AdaptiveETA(), ']' ]) for i, _ in enumerate(sampler.sample(p0, iterations=n_iterations)): bar.update(i + 1) result['samples'] = sampler.chain[:, n_burnin:, :] \ .reshape((-1, dim)).T if self._fix_k: result['samples'][0, :] = log(self._fix_k) if self._fix_p: result['samples'][1, :] = log(self._fix_p) self.params = { k: { 'k': exp(data[0]), 'p': exp(data[1]), 'a': data[4], 'b': data[5], 'alpha': data[6:6 + n_features].T, 'beta': data[6 + n_features:6 + 2 * n_features].T, } for k, data in result.items() }
def finish(self, resume=False, verbose=False, use_pbar=True): """Block and complete all threads in queue. Args: resume (bool, optional): Resume spooling after finished verbose (bool, optional): Report progress towards queue completion. use_pbar (bool, optional): Graphically display progress towards queue completions """ # Stop existing spools self.background_spool = False self.dirty.set() # By default, we don't use a callback. cb = None if verbose: print(self) # Progress bar management, optional. self._pbar_max = self.numRunningThreads + len(self.queue) if use_pbar and self._pbar_max > 0: widgets = [ ("[{n}] ".format(n=self.name) if self.name else ''), progressbar.Percentage(), ' ', progressbar.SimpleProgress(format='%(value_s)s of %(max_value_s)s'), ' ', progressbar.Bar(), ' ', DynamicProgressString(name="state"), ' ', progressbar.Timer(), ' ', progressbar.AdaptiveETA(), ] self.progbar = progbar = progressbar.ProgressBar(max_value=self._pbar_max, widgets=widgets, redirect_stdout=True) def updateProgrssBar(): # Update progress bar. q = (len(self.queue) if self.queue else 0) nrt = self.numRunningThreads progress = (self._pbar_max - (nrt + q)) state = "[Spool: Q: {q:2}, R: {nrt:2}/{quota}]".format(quota=self.quota, **locals()) progbar.max_value = self._pbar_max progbar.update(progress, state=state) cb = updateProgrssBar # assert not self.spoolThread.isAlive, "Background loop did not terminate" # Create a spoolloop, but block until it deploys all threads. execif(cb) while (self.queue and len(self.queue) > 0) or (self.numRunningThreads > 0): self.dirty.wait() self.doSpool(verbose=verbose) self.dirty.clear() execif(cb) assert len(self.queue) == 0, "Finished without deploying all threads" assert self.numRunningThreads == 0, "Finished without finishing all threads" if cb: progbar.finish() if resume: self.queue.clear() # Create a fresh queue self.start() if verbose: print(self)
[ "Maven build: ", get_colour(Fore.YELLOW), progressbar.Percentage(), get_colour(Fore.RESET), " ", progressbar.Counter(format='(%(value)d of %(max_value)d)'), get_colour(Fore.LIGHTGREEN_EX), progressbar.Bar(marker="\u2588"), get_colour(Fore.RESET), " ", progressbar.Timer(), " ", get_colour(Fore.MAGENTA), progressbar.AbsoluteETA(format='Finishes: %(eta)s', format_finished='Finished at %(eta)s') if absolute_time else progressbar.AdaptiveETA(), get_colour(Fore.RESET) ] def ansi_length(o): ansi_occ = re.findall(r'\x1B\[[0-?]*[ -/]*[@-~]', o) ansi_len = 0 for occ in ansi_occ: ansi_len += len(occ) return len(o) - ansi_len def match(): count = 0 bar = None
def processingLayer3(self): """ Processing for RGB image using Fourier transform to aproach the segmentation """ print(" ") print("Preprocessing - Layer 3 processing") f = np.fft.fft2(self.image) fshift = np.fft.fftshift(f) magnitude_spectrum = (20 * np.log(np.abs(fshift))).astype('uint8') ## Mean procedure mean_fourier = np.array([ magnitude_spectrum[:, :, 0].mean(), magnitude_spectrum[:, :, 1].mean(), magnitude_spectrum[:, :, 2].mean() ]) ## Range procedure hist_01 = cv2.calcHist([magnitude_spectrum], [0, 1], None, [256, 256], [0, 256, 0, 256]) hist_02 = cv2.calcHist([magnitude_spectrum], [0, 2], None, [256, 256], [0, 256, 0, 256]) val_01 = np.percentile(hist_01, self.percentFou) val_02 = np.percentile(hist_02, self.percentFou) mask_01 = hist_01 > val_01 mask_02 = hist_02 > val_02 cnts_01, _ = cv2.findContours((mask_01.copy()).astype('uint8'), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) cnts_02, _ = cv2.findContours((mask_02.copy()).astype('uint8'), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) for c in cnts_01: (x1, y1, w1, h1) = cv2.boundingRect(np.array(c)) for h in cnts_02: (x2, y2, w2, h2) = cv2.boundingRect(np.array(h)) ### Range fourier rangeFourier = np.array([[x2, x2 + w2], [x1, x1 + w1], [y1, y1 + h1]]) ##### Procesing of the image uding mean aproach fourier = np.copy(magnitude_spectrum) image_trasnform = np.copy(self.image) size_x, size_y, size_c = fourier.shape binary_fourier = np.zeros([size_x, size_y], dtype='uint8') val = np.zeros(size_c) ### Utilizando media de color percent = 0.83 inicio = time.time() area = size_x * size_y widgets = [ progressbar.Percentage(), ' ', progressbar.Bar(), ' ', progressbar.ETA(), ' ', progressbar.AdaptiveETA() ] bar = progressbar.ProgressBar(widgets=widgets, maxval=area) bar.start() for x in range(0, size_x): for y in range(0, size_y): for c in range(0, size_c): val[c] = np.abs(1 - ( (image_trasnform[x, y, c]) - mean_fourier[c]) / 255) if (val[c] < percent): val[0] = 0 break elif (val[c] >= percent): val[c] = 1 #se cumple la condición de rango se deja el pixel if (val[0] == 0): binary_fourier[x, y] = 0 elif (val.all() == 1): binary_fourier[x, y] = 1 val = np.zeros(size_c) fraction = x * y bar.update(fraction) final = time.time() - inicio bar.update(area) print("Tiempo de procesamiento : ", round(final, 2), "Segundos") ### Morphological Process # First do a dilatation radio = 2 sel = disk(radio) binary_dilat1 = dilation(binary_fourier, sel) for i in range(0, 2): binary_dilat1 = dilation(binary_dilat1, sel) # Second erase little objects radio = 5 sel = disk(radio) binary_erosion1 = erosion(binary_dilat1.copy(), sel) for i in range(0, 2): binary_erosion1 = erosion(binary_erosion1, sel) # Then dilate again radio = 3 sel = disk(radio) binary = dilation(binary_erosion1.copy(), sel) return binary, binary_fourier, rangeFourier, mean_fourier
def analyze_data(scan_data_filenames, ignore_columns, fei4b=False): logging.info("Analyzing and plotting results...") output_h5_filename = local_configuration['output_data_filename'] + '.h5' logging.info('Saving calibration in: %s' % output_h5_filename) if local_configuration['create_plots'] or local_configuration['create_result_plots']: output_pdf_filename = local_configuration['output_data_filename'] + '.pdf' logging.info('Saving plots in: %s' % output_pdf_filename) output_pdf = PdfPages(output_pdf_filename) # define output data structures mean_threshold_calibration = np.zeros(shape=(len(local_configuration['delays']),), dtype='<f8') # array to hold the analyzed data in ram mean_threshold_rms_calibration = np.zeros_like(mean_threshold_calibration) # array to hold the analyzed data in ram mean_noise_calibration = np.zeros_like(mean_threshold_calibration) # array to hold the analyzed data in ram mean_noise_rms_calibration = np.zeros_like(mean_threshold_calibration) # array to hold the analyzed data in ram threshold_calibration = np.zeros(shape=(80, 336, len(local_configuration['delays'])), dtype='<f8') # array to hold the analyzed data in ram noise_calibration = np.zeros_like(threshold_calibration) # array to hold the analyzed data in ram mean_threshold_calibration_1 = np.zeros_like(mean_threshold_calibration) # array to hold the analyzed data in ram mean_threshold_rms_calibration_1 = np.zeros_like(mean_threshold_calibration) # array to hold the analyzed data in ram threshold_calibration_1 = np.zeros_like(threshold_calibration) # array to hold the analyzed data in ram mean_threshold_calibration_2 = np.zeros_like(mean_threshold_calibration) # array to hold the analyzed data in ram mean_threshold_rms_calibration_2 = np.zeros_like(mean_threshold_calibration) # array to hold the analyzed data in ram threshold_calibration_2 = np.zeros_like(threshold_calibration) # array to hold the analyzed data in ram # initialize progress bar progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=len(local_configuration['delays']), term_width=80) progress_bar.start() # loop over all delay values and analyze the corresponding data for delay_index, delay_value in enumerate(local_configuration['delays']): # interpret the raw data from the actual delay value raw_data_file = scan_data_filenames[delay_value] analyzed_data_file = os.path.splitext(raw_data_file)[0] + '_interpreted.h5' analyze(raw_data_file=raw_data_file, analyzed_data_file=analyzed_data_file, fei4b=fei4b) scan_parameters = None with tb.open_file(analyzed_data_file, mode="r") as in_file_h5: # mask the not scanned columns for analysis and plotting mask = np.logical_or(mask_columns(pixel_array=in_file_h5.root.HistThresholdFitted[:], ignore_columns=ignore_columns), mask_pixel(steps=3, shift=0).T) == 0 occupancy_masked = mask_columns(pixel_array=in_file_h5.root.HistOcc[:], ignore_columns=ignore_columns) thresholds_masked = np.ma.masked_array(in_file_h5.root.HistThresholdFitted[:], mask) noise_masked = np.ma.masked_array(in_file_h5.root.HistNoiseFitted[:], mask) # plot the threshold distribution and the s curves if local_configuration['create_plots']: plotting.plot_three_way(hist=thresholds_masked * 55.0, title='Threshold Fitted for delay = ' + str(delay_value), x_axis_title='threshold [e]', filename=output_pdf) plotting.plot_relative_bcid(hist=in_file_h5.root.HistRelBcid[0:16], title='Relative BCID (former LVL1ID) for delay = ' + str(delay_value), filename=output_pdf) plotting.plot_event_errors(hist=in_file_h5.root.HistErrorCounter[:], title='Event status for delay = ' + str(delay_value), filename=output_pdf) meta_data_array = in_file_h5.root.meta_data[:] parameter_settings = analysis_utils.get_scan_parameter(meta_data_array=meta_data_array) scan_parameters = parameter_settings['PlsrDAC'] if local_configuration['create_plots']: plotting.plot_scurves(occupancy_hist=occupancy_masked, title='S-Curves, delay ' + str(delay_value), scan_parameters=scan_parameters, scan_parameter_name='PlsrDAC', filename=output_pdf) # fill the calibration data arrays mean_threshold_calibration[delay_index] = np.ma.mean(thresholds_masked) mean_threshold_rms_calibration[delay_index] = np.ma.std(thresholds_masked) threshold_calibration[:, :, delay_index] = thresholds_masked.T mean_noise_calibration[delay_index] = np.ma.mean(noise_masked) mean_noise_rms_calibration[delay_index] = np.ma.std(noise_masked) noise_calibration[:, :, delay_index] = noise_masked.T # if activated analyze also the trigger seperately if local_configuration['analysis_two_trigger']: with tb.open_file(os.path.splitext(analyzed_data_file)[0] + '_analyzed_1.h5', mode="r") as in_file_1_h5: with tb.open_file(os.path.splitext(analyzed_data_file)[0] + '_analyzed_2.h5', mode="r") as in_file_2_h5: # mask the not scanned columns for analysis and plotting try: occupancy_masked_1 = occupancy_masked = mask_columns(pixel_array=in_file_1_h5.root.HistOcc[:], ignore_columns=ignore_columns) thresholds_masked_1 = np.ma.masked_array(in_file_1_h5.root.HistThresholdFitted[:], mask) rel_bcid_1 = in_file_1_h5.root.HistRelBcid[0:16] except tb.exceptions.NoSuchNodeError: occupancy_masked_1 = np.zeros(shape=(336, 80, 2)) thresholds_masked_1 = np.zeros(shape=(336, 80)) rel_bcid_1 = np.zeros(shape=(16, )) try: occupancy_masked_2 = occupancy_masked = mask_columns(pixel_array=in_file_2_h5.root.HistOcc[:], ignore_columns=ignore_columns) thresholds_masked_2 = np.ma.masked_array(in_file_2_h5.root.HistThresholdFitted[:], mask) rel_bcid_2 = in_file_2_h5.root.HistRelBcid[0:16] except tb.exceptions.NoSuchNodeError: occupancy_masked_2 = np.zeros(shape=(336, 80, 2)) thresholds_masked_2 = np.zeros(shape=(336, 80)) rel_bcid_2 = np.zeros(shape=(16, )) # plot the threshold distribution and the s curves if local_configuration['create_plots']: plotting.plot_three_way(hist=thresholds_masked_1 * 55.0, title='Threshold Fitted for 1. trigger, delay ' + str(delay_value), x_axis_title='threshold [e]', filename=output_pdf) plotting.plot_relative_bcid(hist=rel_bcid_1, title='Relative BCID (former LVL1ID) for 1. trigger, delay = ' + str(delay_value), filename=output_pdf) plotting.plot_three_way(hist=thresholds_masked_2 * 55.0, title='Threshold Fitted for 2. trigger, delay ' + str(delay_value), x_axis_title='threshold [e]', filename=output_pdf) plotting.plot_relative_bcid(hist=rel_bcid_2, title='Relative BCID (former LVL1ID) for 2. trigger, delay = ' + str(delay_value), filename=output_pdf) if local_configuration['create_plots']: plotting.plot_scurves(occupancy_hist=occupancy_masked_1, title='S-Curves 1. trigger, delay ' + str(delay_value), scan_parameters=scan_parameters, scan_parameter_name='PlsrDAC', filename=output_pdf) plotting.plot_scurves(occupancy_hist=occupancy_masked_2, title='S-Curves 2. trigger, delay ' + str(delay_value), scan_parameters=scan_parameters, scan_parameter_name='PlsrDAC', filename=output_pdf) # fill the calibration data arrays mean_threshold_calibration_1[delay_index] = np.ma.mean(thresholds_masked_1) mean_threshold_rms_calibration_1[delay_index] = np.ma.std(thresholds_masked_1) threshold_calibration_1[:, :, delay_index] = thresholds_masked_1.T mean_threshold_calibration_2[delay_index] = np.ma.mean(thresholds_masked_2) mean_threshold_rms_calibration_2[delay_index] = np.ma.std(thresholds_masked_2) threshold_calibration_2[:, :, delay_index] = thresholds_masked_2.T progress_bar.update(delay_index) progress_bar.finish() # plot the parameter against delay plots if local_configuration['create_result_plots']: plotting.plot_scatter(x=local_configuration['delays'], y=mean_threshold_calibration * 55.0, title='Threshold as a function of the delay', x_label='delay [BCID]', y_label='Mean threshold [e]', log_x=False, filename=output_pdf) plotting.plot_scatter(x=local_configuration['delays'], y=mean_threshold_calibration * 55.0, title='Threshold as a function of the delay', x_label='delay [BCID]', y_label='Mean threshold [e]', log_x=True, filename=output_pdf) plotting.plot_scatter(x=local_configuration['delays'], y=mean_threshold_rms_calibration * 55.0, title='Threshold as a function of the delay', x_label='delay [BCID]', y_label='Threshold RMS [e]', log_x=False, filename=output_pdf) plotting.plot_scatter(x=local_configuration['delays'], y=mean_threshold_rms_calibration * 55.0, title='Threshold as a function of the delay', x_label='delay [BCID]', y_label='Threshold RMS [e]', log_x=True, filename=output_pdf) if local_configuration['analysis_two_trigger']: plotting.plot_scatter(x=local_configuration['delays'], y=mean_threshold_calibration_1 * 55.0, title='Threshold as a function of the delay, 1. trigger', x_label='delay [BCID]', y_label='Mean threshold [e]', log_x=False, filename=output_pdf) plotting.plot_scatter(x=local_configuration['delays'], y=mean_threshold_calibration_1 * 55.0, title='Threshold as a function of the delay, 1. trigger', x_label='delay [BCID]', y_label='Mean threshold [e]', log_x=True, filename=output_pdf) plotting.plot_scatter(x=local_configuration['delays'], y=mean_threshold_rms_calibration_1 * 55.0, title='Threshold as a function of the delay, 1. trigger', x_label='delay [BCID]', y_label='Threshold RMS [e]', log_x=False, filename=output_pdf) plotting.plot_scatter(x=local_configuration['delays'], y=mean_threshold_rms_calibration_1 * 55.0, title='Threshold as a function of the delay, 1. trigger', x_label='delay [BCID]', y_label='Threshold RMS [e]', log_x=True, filename=output_pdf) plotting.plot_scatter(x=local_configuration['delays'], y=mean_threshold_calibration_2 * 55.0, title='Threshold as a function of the delay, 2. trigger', x_label='delay [BCID]', y_label='Mean threshold [e]', log_x=False, filename=output_pdf) plotting.plot_scatter(x=local_configuration['delays'], y=mean_threshold_calibration_2 * 55.0, title='Threshold as a function of the delay, 2. trigger', x_label='delay [BCID]', y_label='Mean threshold [e]', log_x=True, filename=output_pdf) plotting.plot_scatter(x=local_configuration['delays'], y=mean_threshold_rms_calibration_2 * 55.0, title='Threshold as a function of the delay, 2. trigger', x_label='delay [BCID]', y_label='Threshold RMS [e]', log_x=False, filename=output_pdf) plotting.plot_scatter(x=local_configuration['delays'], y=mean_threshold_rms_calibration_2 * 55.0, title='Threshold as a function of the delay, 2. trigger', x_label='delay [BCID]', y_label='Threshold RMS [e]', log_x=True, filename=output_pdf) plotting.plot_scatter(x=local_configuration['delays'], y=mean_noise_calibration * 55.0, title='Noise as a function of the delay', x_label='delay [BCID]', y_label='Mean noise [e]', log_x=False, filename=output_pdf) plotting.plot_scatter(x=local_configuration['delays'], y=mean_noise_rms_calibration * 55.0, title='Noise as a function of the delay', x_label='delay [BCID]', y_label='Noise RMS [e]', log_x=False, filename=output_pdf) if local_configuration['create_plots'] or local_configuration['create_result_plots']: output_pdf.close() # store the calibration data into a hdf5 file as an easy to read table and as an array for quick data access with tb.open_file(output_h5_filename, mode="w") as out_file_h5: store_calibration_data_as_array(out_file_h5, mean_threshold_calibration, mean_threshold_rms_calibration, threshold_calibration, mean_noise_calibration, mean_noise_rms_calibration, noise_calibration) store_calibration_data_as_table(out_file_h5, mean_threshold_calibration, mean_threshold_rms_calibration, threshold_calibration, mean_noise_calibration, mean_noise_rms_calibration, noise_calibration)
import logging import datetime import warnings import traceback import io import pprint import inspect import numpy import xarray import progressbar #: progressbar widget used to display progress for various reading tasks my_pb_widget = [ progressbar.Bar("=", "[", "]"), " ", progressbar.Percentage(), " (", progressbar.AdaptiveETA(), " -> ", progressbar.AbsoluteETA(), ') ' ] def add_to_argparse(parser, include_period=True, include_sat=0, include_channels=True, include_temperatures=False, include_debug=False, include_version=False): """Add commoners to argparse object. Helper function to add flags to :class:`argparse.ArgumentParser` objects, where the exact same flags are occurring in multiple scripts.
def select_trigger_hits(input_file_hits, output_file_hits_1, output_file_hits_2): if (not os.path.isfile(output_file_hits_1) and not os.path.isfile(output_file_hits_2)) or local_configuration['overwrite_output_files']: with tb.open_file(input_file_hits, mode="r") as in_hit_file_h5: hit_table_in = in_hit_file_h5.root.Hits with tb.open_file(output_file_hits_1, mode="w") as out_hit_file_1_h5: with tb.open_file(output_file_hits_2, mode="w") as out_hit_file_2_h5: hit_table_out_1 = out_hit_file_1_h5.create_table(out_hit_file_1_h5.root, name='Hits', description=data_struct.HitInfoTable, title='hit_data', filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False)) hit_table_out_2 = out_hit_file_2_h5.create_table(out_hit_file_2_h5.root, name='Hits', description=data_struct.HitInfoTable, title='hit_data', filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False)) progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=hit_table_in.shape[0], term_width=80) progress_bar.start() for data, index in analysis_utils.data_aligned_at_events(hit_table_in, chunk_size=5000000): hit_table_out_1.append(data[data['LVL1ID'] % 2 == 1]) # first trigger hits hit_table_out_2.append(data[data['LVL1ID'] % 2 == 0]) # second trigger hits progress_bar.update(index) progress_bar.finish() in_hit_file_h5.root.meta_data.copy(out_hit_file_1_h5.root) # copy meta_data note to new file in_hit_file_h5.root.meta_data.copy(out_hit_file_2_h5.root) # copy meta_data note to new file
def scan(self): # Stop mode related hacks to read all hits stored with stop mode self.register.set_global_register_value("StopModeCnfg", 1) stop_mode_cmd = self.register.get_commands("WrRegister", name=["StopModeCnfg"])[0] self.register.set_global_register_value("StopModeCnfg", 0) stop_mode_off_cmd = self.register.get_commands("WrRegister", name=["StopModeCnfg" ])[0] self.register.set_global_register_value("StopClkPulse", 1) stop_clock_pulse_cmd_high = self.register.get_commands( "WrRegister", name=["StopClkPulse"])[0] self.register.set_global_register_value("StopClkPulse", 0) stop_clock_pulse_cmd_low = self.register.get_commands( "WrRegister", name=["StopClkPulse"])[0] start_sequence = self.register_utils.concatenate_commands(( self.register.get_commands("zeros", length=self.trigger_delay)[0], stop_mode_cmd, self.register.get_commands("zeros", length=20)[0], stop_clock_pulse_cmd_high, # FIXME: before ConfMode? self.register.get_commands("zeros", length=50)[0], self.register.get_commands("ConfMode")[0])) stop_sequence = self.register_utils.concatenate_commands( (self.register.get_commands("zeros", length=50)[0], stop_clock_pulse_cmd_low, self.register.get_commands("zeros", length=10)[0], stop_mode_off_cmd, self.register.get_commands("zeros", length=400)[0])) # define the command sequence to read the hits of one latency count one_latency_read = self.register_utils.concatenate_commands( (self.register.get_commands("zeros", length=50)[0], self.register.get_commands("RunMode")[0], self.register.get_commands("zeros", length=50)[0], self.register.get_commands("LV1")[0], self.register.get_commands("zeros", length=self.readout_delay)[0], self.register.get_commands("ConfMode")[0], self.register.get_commands("zeros", length=1000)[0], self.register.get_commands("GlobalPulse", Width=0)[0], self.register.get_commands("zeros", length=100)[0])) self.dut['TX']['CMD_REPEAT'] = self.trig_count self.dut['TX']['START_SEQUENCE_LENGTH'] = len(start_sequence) self.dut['TX']['STOP_SEQUENCE_LENGTH'] = len(stop_sequence) + 1 # preload the command to be send for each trigger command = self.register_utils.concatenate_commands( (start_sequence, one_latency_read, stop_sequence)) self.register_utils.set_command(command) with self.readout(no_data_timeout=self.no_data_timeout, **self.scan_parameters._asdict()): with self.trigger(): got_data = False start = time() while not self.stop_run.wait(1.0): if not got_data: if self.data_words_per_second() > 0: got_data = True logging.info('Taking data...') if self.max_triggers: self.progressbar = progressbar.ProgressBar( widgets=[ '', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA() ], maxval=self.max_triggers, poll=10, term_width=80).start() else: self.progressbar = progressbar.ProgressBar( widgets=[ '', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.Timer() ], maxval=self.scan_timeout, poll=10, term_width=80).start() else: triggers = self.dut['TLU']['TRIGGER_COUNTER'] try: if self.max_triggers: self.progressbar.update(triggers) else: self.progressbar.update(time() - start) except ValueError: pass if self.max_triggers and triggers >= self.max_triggers: self.progressbar.finish() self.stop(msg='Trigger limit was reached: %i' % self.max_triggers) logging.info('Total amount of triggers collected: %d', self.dut['TLU']['TRIGGER_COUNTER'])
def transfer_file( file_name, socket_addr ): # Function to open the raw data file and sending the readouts periodically context = zmq.Context() socket = context.socket(zmq.PUSH) socket.connect(socket_addr) # change to socket.bind in case ob PUB /SUB # recv_socket = context.socket(zmq.PULL) # recv_socket.connect('tcp://127.0.0.1:5011') logging.info("data sent to %s" % socket_addr) with tb.open_file(file_name, mode="r") as in_file_h5: start = time.time() meta_data = in_file_h5.root.meta_data[:] raw_data = in_file_h5.root.raw_data[:] n_readouts = meta_data.shape[0] last_readout_time = time.time() try: scan_parameter_names = in_file_h5.root.configuration.conf[:][ 'name'] #in_file_h5.root.scan_parameters.dtype.names except tb.NoSuchNodeError: scan_parameter_names = None progress_bar = progressbar.ProgressBar(widgets=[ '', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA() ], maxval=meta_data.shape[0], term_width=80) progress_bar.start() for i in range(n_readouts): # Raw data indeces of readout i_start = meta_data['index_start'][i] i_stop = meta_data['index_stop'][i] # Time stamps of readout t_stop = meta_data[i]['timestamp_stop'] t_start = meta_data[i]['timestamp_start'] # Create data of readout (raw data + meta data) data = [] data.append(raw_data[i_start:i_stop]) data.extend( (float(t_start), float(t_stop), int(meta_data[i]['error']))) # scan_par_id = int(meta_data[i]['scan_param_id']) send_data(socket, data) if i == 0: # Initialize on first readout last_timestamp_start = t_start now = time.time() # if now - start > 180: # break delay = now - last_readout_time additional_delay = t_start - last_timestamp_start - delay if additional_delay > 0: # Wait if send too fast, especially needed when readout was # stopped during data taking (e.g. for mask shifting) time.sleep(additional_delay) last_readout_time = time.time() last_timestamp_start = t_start time.sleep(meta_data[i]['timestamp_stop'] - meta_data[i]['timestamp_start']) progress_bar.update(i) progress_bar.finish() socket.close() context.term()