def process_data(self, filename, max_events=None): #Open output hdf5 file f = tables.open_file(self.output_path, mode="a", title="Output File") # Open Root File root_file = VARootFile(filename) self.write_metadata(f, root_file) selected_tels, num_tel = self.select_telescopes(root_file) arr_info = root_file.get_array_info() if not f.__contains__('/Array_Info'): arr_table = f.create_table(f.root, 'Array_Info', row_types.Array, ("Table of array data")) arr_row = arr_table.row for tel_type in selected_tels: for tel_id in selected_tels[tel_type]: arr_row["tel_id"] = arr_info[tel_id]['tel_id'] arr_row["tel_x"] = arr_info[tel_id]['tel_x'] arr_row["tel_y"] = arr_info[tel_id]['tel_y'] arr_row["tel_z"] = arr_info[tel_id]['tel_z'] arr_row["tel_type"] = arr_info[tel_id]['tel_type'] arr_row["run_array_direction"] = arr_info[tel_id][ 'run_array_direction'] arr_row.append() if not f.__contains__('/Telescope_Info'): tel_table = f.create_table(f.root, 'Telescope_Info', row_types.Tel, ("Table of telescope data")) descr = tel_table.description._v_colobjects descr2 = descr.copy() if (self.one_D_image_oversampled): logger.debug('Save dummy pixel pos') descr2["pixel_pos"] = tables.Float32Col(shape=(2, 54 * 54)) tel_table2 = f.create_table(f.root, 'temp', descr2, "Table of telescope data") tel_table.attrs._f_copy(tel_table2) tel_table.remove() tel_table2.move(f.root, 'Telescope_Info') tel_row = tel_table2.row # add units to table attributes random_tel_type = "VTS" random_tel_id = 0 tel_table2.attrs.tel_pos_units = str("aru") tel_row['tel_type'] = "VTS" posx, posy = np.meshgrid(np.arange(0, 54, dtype='float32'), np.arange(0, 54, dtype='float32')) tel_row['pixel_pos'] = [posx.ravel(), posy.ravel()] tel_row.append() else: logger.debug('Save real pixel pos') descr2["pixel_pos"] = tables.Float32Col(shape=(2, 499)) tel_table2 = f.create_table(f.root, 'temp', descr2, "Table of telescope data") tel_table.attrs._f_copy(tel_table2) tel_table.remove() tel_table2.move(f.root, 'Telescope_Info') tel_row = tel_table2.row # add units to table attributes random_tel_type = "VTS" random_tel_id = 0 tel_table2.attrs.tel_pos_units = str("mm") tel_row['tel_type'] = "VTS" pos = np.loadtxt(data_dir + '/pixel_position.txt') tel_row['pixel_pos'] = [pos[:, 0], pos[:, 1]] tel_row.append() #create event table if not f.__contains__('/Event_Info'): table = f.create_table(f.root, 'Event_Info', row_types.Event, "Table of Event metadata") descr = table.description._v_colobjects descr2 = descr.copy() if self.storage_mode == 'tel_type': for tel_type in selected_tels: descr2[tel_type + '_indices'] = tables.Int32Col( shape=(len(selected_tels[tel_type]))) elif self.storage_mode == 'tel_id': descr2["indices"] = tables.Int32Col(shape=(num_tel)) table2 = f.create_table(f.root, 'temp', descr2, "Table of Events") table.attrs._f_copy(table2) table.remove() table2.move(f.root, 'Event_Info') #add units to table attributes table2.attrs.core_pos_units = 'm' table2.attrs.h_first_int_units = 'm' table2.attrs.mc_energy_units = 'TeV' table2.attrs.alt_az_units = 'rad' #create image tables for tel_type in selected_tels: if self.img_mode == '2D': img_width = self.IMAGE_SHAPES[tel_type][0] img_length = self.IMAGE_SHAPES[tel_type][1] if self.img_dim_order == 'channels_first': array_shape = (1, img_width, img_length) elif self.img_dim_order == 'channels_last': array_shape = (img_width, img_length, 1) np_type = np.dtype( (np.dtype(self.img_dtypes[tel_type]), array_shape)) columns_dict = { "image": tables.Col.from_dtype(np_type), "event_index": tables.Int32Col() } elif self.img_mode == '1D': array_shape = (image.TEL_NUM_PIXELS_OVER_SAMPLED[tel_type], ) if self.one_D_image_oversampled else ( image.TEL_NUM_PIXELS[tel_type], ) np_type = np.dtype( (np.dtype(self.img_dtypes[tel_type]), array_shape)) columns_dict = { "image_charge": tables.Col.from_dtype(np_type), "event_index": tables.Int32Col(), "image_peak_times": tables.Col.from_dtype(np_type) } description = type('description', (tables.IsDescription, ), columns_dict) if self.storage_mode == 'tel_type': if not f.__contains__('/' + tel_type): table = f.create_table( f.root, tel_type, description, "Table of {} images".format(tel_type)) #append blank image at index 0 image_row = table.row if self.img_mode == '2D': image_row['image'] = self.trace_converter.convert( np.zeros(500)) elif self.img_mode == '1D': shape = (image.TEL_NUM_PIXELS_OVER_SAMPLED[tel_type], ) if self.one_D_image_oversampled else ( image.TEL_NUM_PIXELS[tel_type], ) image_row['image_charge'] = np.zeros( shape, dtype=self.img_dtypes[tel_type]) image_row['image_peak_times'] = np.zeros( shape, dtype=self.img_dtypes[tel_type]) image_row['event_index'] = -1 image_row.append() table.flush() elif self.storage_mode == 'tel_id': for tel_id in selected_tels[tel_type]: if not f.__contains__('/T' + str(tel_id)): table = f.create_table( f.root, 'T' + str(tel_id), description, "Table of T{} images".format(str(tel_id))) #append blank image at index 0 image_row = table.row if self.img_mode == '2D': image_row['image'] = self.trace_converter.convert( np.zeros(500)) elif self.img_mode == '1D': shape = ( image.TEL_NUM_PIXELS_OVER_SAMPLED[tel_type], ) if self.one_D_image_oversampled else ( image.TEL_NUM_PIXELS[tel_type], ) image_row['image_charge'] = np.zeros( shape, dtype=self.img_dtypes[tel_type]) image_row['image_peak_times'] = np.zeros( shape, dtype=self.img_dtypes[tel_type]) image_row['event_index'] = -1 image_row.append() table.flush() # specify calibration and other processing options logger.info("Processing events...") event_count = 0 passing_count = 0 if (max_events is None): source = root_file.read_st2_calib_channel_charge( tels=[i - 1 for i in selected_tels[tel_type]], cleaning=None) else: source = root_file.read_st2_calib_channel_charge( tels=[i - 1 for i in selected_tels[tel_type]], cleaning=None, stop_event=max_events) for i, simData, event, tzero, triggeredTels in source: if ((max_events is not None) and (event_count > max_events)): break event_count += 1 table = f.root.Event_Info table.flush() event_row = table.row event_index = table.nrows if self.storage_mode == 'tel_type': tel_index_vectors = { tel_type: [] for tel_type in selected_tels } elif self.storage_mode == 'tel_id': all_tel_index_vector = [] for tel_type in selected_tels.keys(): for tel_id in sorted(selected_tels[tel_type]): if self.storage_mode == 'tel_type': index_vector = tel_index_vectors[tel_type] elif self.storage_mode == 'tel_id': index_vector = all_tel_index_vector if (self.force_all_telescopes): triggeredTels = [1, 2, 3, 4] if tel_id in triggeredTels: pixel_vector = event[tel_id - 1, :] timing_vector = tzero[tel_id - 1, :] logger.debug( 'Storing image from tel_type {} ({} pixels)'. format(tel_type, len(pixel_vector))) if self.storage_mode == 'tel_type': table = eval('f.root.{}'.format(tel_type)) elif self.storage_mode == 'tel_id': table = eval('f.root.T{}'.format(tel_id)) next_index = table.nrows image_row = table.row imgs = self.trace_converter.convert(pixel_vector) time_imgs = self.trace_converter.convert( timing_vector) * 4 if self.img_mode == '2D': image_row['image'] = imgs elif self.img_mode == '1D': if (self.one_D_image_oversampled): image_row['image_charge'] = imgs.reshape( array_shape).copy() image_row[ 'image_peak_times'] = time_imgs.reshape( array_shape).copy() else: image_row[ 'image_charge'] = pixel_vector[:499].copy( ) image_row[ 'image_peak_times'] = timing_vector[: 499].copy( ) image_row["event_index"] = event_index image_row.append() index_vector.append(next_index) table.flush() else: index_vector.append(0) if self.storage_mode == 'tel_type': for tel_type in tel_index_vectors: event_row[tel_type + '_indices'] = tel_index_vectors[tel_type] elif self.storage_mode == 'tel_id': event_row['indices'] = all_tel_index_vector event_row['event_number'] = i event_row['run_number'] = simData.fRunNum event_row['particle_id'] = simData.fCORSIKAParticleID event_row['core_x'] = simData.fCoreEastM event_row['core_y'] = simData.fCoreSouthM * -1 event_row['h_first_int'] = 0 event_row['mc_energy'] = simData.fEnergyGeV / 1000. event_row['alt'] = simData.fPrimaryZenithDeg * np.pi * 180. event_row['az'] = simData.fPrimaryAzimuthDeg * np.pi * 180. event_row.append() table.flush() f.root.Event_Info.flush() total_num_events = f.root.Event_Info.nrows f.close() logger.info("{} events read in file".format(event_count)) logger.info("{} total events in output file.".format(total_num_events)) logger.info("Done!")
class Test(tb.IsDescription): ngroup = tb.Int32Col(pos=1) ntable = tb.Int32Col(pos=2) nrow = tb.Int32Col(pos=3)
class Descr(tables.IsDescription): var1 = tables.StringCol(itemsize=4, shape=(), dflt='', pos=0) var2 = tables.BoolCol(shape=(), dflt=False, pos=1) var3 = tables.Int32Col(shape=(), dflt=0, pos=2) var4 = tables.Float64Col(shape=(), dflt=0.0, pos=3)
class TimeSeriesGrid(tb.IsDescription): sat_name = tb.StringCol(10, pos=1) ref_time = tb.StringCol(10, pos=2) date = tb.StringCol(10, pos=3) year = tb.Int32Col(pos=4) month = tb.Int32Col(pos=5)
class TimeSeriesGrid(tb.IsDescription): satname = tb.StringCol(20, pos=1) time1 = tb.Int32Col(pos=2) time2 = tb.Int32Col(pos=3)
class Obs(tables.IsDescription): #date_time = tables.Int32Col(indexed=True, pos=1) #year = tables.Int16Col(indexed=True, pos=2) #yday = tables.Int16Col(indexed=True, pos=3) #hour = tables.Int8Col(indexed=True, pos=4) date_time = tables.Int32Col(pos=1) year = tables.Int16Col(pos=2) yday = tables.Int16Col(pos=3) hour = tables.Int8Col(pos=4) source = tables.StringCol(1, pos=5) type = tables.StringCol(6, pos=6) wind_dir = tables.Int16Col(pos=7) wdir_type = tables.StringCol(1, pos=8) wind_spd = tables.Float32Col(pos=9) cig = tables.UInt16Col(pos=10) cig_tool = tables.StringCol(1, pos=11) cavok = tables.StringCol(1, pos=12) vis = tables.Int32Col(pos=13) vis_var_code = tables.StringCol(1, pos=14) temp = tables.Float32Col(pos=15) dewt = tables.Float32Col(pos=16) pres = tables.Float32Col(pos=17) prec_period = tables.UInt8Col(shape=p_dim, pos=18) prec_depth = tables.Float32Col(shape=p_dim, pos=19) prec_code = tables.UInt8Col(shape=p_dim, pos=20) dur_code = tables.UInt8Col(pos=21) char_code = tables.StringCol(1, pos=22) disc_code = tables.UInt8Col(pos=23) water_dep = tables.Int16Col(pos=24) snow_dim = tables.Int16Col(pos=25) snow_code = tables.UInt8Col(pos=26) liq_dim = tables.Float32Col(pos=27) liq_code = tables.UInt8Col(pos=28) snow_period = tables.UInt8Col(shape=p_dim, pos=29) snow_accu = tables.Int16Col(shape=p_dim, pos=30) accu_cond = tables.UInt8Col(shape=p_dim, pos=31) pres_wx_code = tables.UInt8Col(pos=32) pt_mwx_code = tables.UInt8Col(shape=o_dim, pos=33) pt_mwx_pd = tables.UInt8Col(shape=o_dim, pos=34) pt_awx_code = tables.UInt8Col(shape=o_dim, pos=35) pt_awx_pd = tables.UInt8Col(shape=o_dim, pos=36) rw_vis_dir = tables.Int16Col(pos=37) rw_code = tables.StringCol(1, pos=38) rw_vis_dim = tables.Int16Col(pos=39) cov_code = tables.UInt8Col(shape=c_dim, pos=40) bs_hi_dim = tables.Int32Col(shape=c_dim, pos=41) cloud_code = tables.UInt8Col(shape=c_dim, pos=42) cov_sum_st_code = tables.UInt8Col(shape=c_dim, pos=43) cov_sum_code = tables.UInt8Col(shape=c_dim, pos=44) cov_sum_st_dim = tables.Int32Col(shape=c_dim, pos=45) cov_sum_char_code = tables.UInt8Col(shape=c_dim, pos=46) total_cov_code = tables.UInt8Col(pos=47) total_opa_code = tables.UInt8Col(pos=48) low_cov_code = tables.UInt8Col(pos=49) low_cld_gen_code = tables.UInt8Col(pos=50) low_cld_dim = tables.Int32Col(pos=51) mid_cld_gen_code = tables.UInt8Col(pos=52) hi_cld_gen_code = tables.UInt8Col(pos=53) st_cov_code = tables.UInt8Col(shape=c_dim, pos=54) st_cld_tp_hi = tables.Int32Col(shape=c_dim, pos=55) st_cld_type = tables.UInt8Col(shape=c_dim, pos=56) st_cld_tp_code = tables.UInt8Col(shape=c_dim, pos=57) sun_dur = tables.Int16Col(pos=58) hail_size = tables.Float32Col(pos=59) g2s_code = tables.UInt8Col(pos=60) mint_period = tables.Float32Col(pos=61) mint = tables.Float32Col(pos=62) x_tp_period = tables.Float32Col(shape=o_dim, pos=63) x_tp_code = tables.UInt8Col(shape=o_dim, pos=64) x_tp = tables.Float32Col(shape=o_dim, pos=65) altimeter = tables.Float32Col(pos=66) st_pres = tables.Float32Col(pos=67) pres_tr = tables.UInt8Col(pos=68) pres_chg_3h = tables.Float32Col(pos=69) pres_chg_24h = tables.Float32Col(pos=70) isobar_code = tables.UInt8Col(pos=71) isobar_dim = tables.Int16Col(pos=72) wx_vic_code = tables.UInt8Col(shape=d_dim, pos=73) pres_wxm_code = tables.UInt8Col(shape=d_dim, pos=74) sup_wd_code = tables.UInt8Col(shape=s_dim, pos=75) sup_wd_prd = tables.UInt8Col(shape=s_dim, pos=76) sup_wd_spd = tables.Float32Col(shape=s_dim, pos=77) wd_gust = tables.Float32Col(pos=78)
class unit_descriptor(tables.IsDescription): electrode_number = tables.Int32Col() single_unit = tables.Int32Col() regular_spiking = tables.Int32Col() fast_spiking = tables.Int32Col()
class RunInfo(tb.IsDescription): run_number = tb.Int32Col(shape=(), pos=0)
class Distance(tables.IsDescription): frame = tables.Int32Col(pos=0) distance = tables.Float64Col(pos=1)
class SymbolCount(tb.IsDescription): symbol_root = tb.StringCol(6, pos=1) count = tb.Int32Col(pos=2) # short integer date = tb.Int32Col(pos=3)
class EventInfo(tb.IsDescription): evt_number = tb.Int32Col(shape=(), pos=0) timestamp = tb.UInt64Col(shape=(), pos=1)
class FillModel(pt.IsDescription): timestamp = pt.Time64Col() quantity = pt.Int32Col() cashChange = pt.Float32Col() commission = pt.Float32Col() impactCost = pt.Float32Col()
class DECONV_PARAM(tb.IsDescription): N_BASELINE = tb.Int32Col(pos=0) THR_TRIGGER = tb.Int16Col(pos=1) ACCUM_DISCHARGE_LENGTH = tb.Int16Col(pos=2)
class TrialData(tables.IsDescription): trial_num = tables.Int32Col() trigger = tables.StringCol(26) response = tables.StringCol(26) plot_trigger = tables.Int8Col() plot_response = tables.Int8Col()
class Particle(tb.IsDescription): name = tb.StringCol(16, pos=1) # 16-character String lati = tb.Int32Col(pos=2) # integer longi = tb.Int32Col(pos=3) # integer pressure = tb.Float32Col(pos=4) # float (single-precision) temperature = tb.Float64Col(pos=5) # double (double-precision)
class Small(tb.IsDescription): var1 = tb.StringCol(itemsize=4) var2 = tb.Int32Col() var3 = tb.Float64Col() var4 = tb.BoolCol()
class TrialData(tables.IsDescription): trial_num = tables.Int32Col() session = tables.Int32Col()
class ProcessEvents(object): """Process HiSPARC events to obtain several observables. This class can be used to process a set of HiSPARC events and adds a few observables like particle arrival time and number of particles in the detector to a copy of the event table. """ processed_events_description = { 'event_id': tables.UInt32Col(pos=0), 'timestamp': tables.Time32Col(pos=1), 'nanoseconds': tables.UInt32Col(pos=2), 'ext_timestamp': tables.UInt64Col(pos=3), 'data_reduction': tables.BoolCol(pos=4), 'trigger_pattern': tables.UInt32Col(pos=5), 'baseline': tables.Int16Col(pos=6, shape=4, dflt=-1), 'std_dev': tables.Int16Col(pos=7, shape=4, dflt=-1), 'n_peaks': tables.Int16Col(pos=8, shape=4, dflt=-1), 'pulseheights': tables.Int16Col(pos=9, shape=4, dflt=-1), 'integrals': tables.Int32Col(pos=10, shape=4, dflt=-1), 'traces': tables.Int16Col(pos=11, shape=(4, 80), dflt=-1), 'event_rate': tables.Float32Col(pos=12), 't1': tables.Float32Col(pos=13, dflt=-1), 't2': tables.Float32Col(pos=14, dflt=-1), 't3': tables.Float32Col(pos=15, dflt=-1), 't4': tables.Float32Col(pos=16, dflt=-1), 'n1': tables.Float32Col(pos=17, dflt=-1), 'n2': tables.Float32Col(pos=18, dflt=-1), 'n3': tables.Float32Col(pos=19, dflt=-1), 'n4': tables.Float32Col(pos=20, dflt=-1), 't_trigger': tables.Float32Col(pos=21, dflt=-1) } def __init__(self, data, group, source=None, progress=True): """Initialize the class. :param data: the PyTables datafile :param group: the group containing the station data. In normal cases, this is simply the group containing the events table. :param source: the name of the events table. Default: None, meaning the default name 'events'. :param progress: if True show a progressbar while copying and processing events. """ self.data = data self.group = data.get_node(group) self.source = self._get_source(source) self.progress = progress self.limit = None def process_and_store_results(self, destination=None, overwrite=False, limit=None): """Process events and store the results. :param destination: name of the table where the results will be written. The default, None, corresponds to 'events'. :param overwrite: if True, overwrite previously obtained results. :param limit: the maximum number of events that will be stored. The default, None, corresponds to no limit. """ self.limit = limit self._check_destination(destination, overwrite) self._clean_events_table() self._create_results_table() self._store_results_from_traces() self._store_number_of_particles() self._move_results_table_into_destination() def get_traces_for_event(self, event): """Return the traces from an event. :param event: a row from the events table. :return: the traces: an array of pulseheight values. """ traces = [ list(self._get_trace(idx)) for idx in event['traces'] if idx >= 0 ] # Make traces follow NumPy conventions traces = np.array(traces).T return traces def get_traces_for_event_index(self, idx): """Return the traces from event #idx. :param idx: the index number of the event. :return: the traces: an array of pulseheight values. """ event = self.source[idx] return self.get_traces_for_event(event) def _get_source(self, source): """Return the table containing the events. :param source: the *name* of the table. If None, this method will try to find the original events table, even if the events were previously processed. :return: table object """ if source is None: if '_events' in self.group: source = self.group._events else: source = self.group.events else: source = self.data.get_node(self.group, source) return source def _check_destination(self, destination, overwrite): """Check if the destination is valid""" if destination == '_events': raise RuntimeError("The _events table is reserved for internal " "use. Choose another destination.") elif destination is None: destination = 'events' # If destination == source, source will be moved out of the way. Don't # worry. Otherwise, destination may not exist or will be overwritten if self.source.name != destination: if destination in self.group and not overwrite: raise RuntimeError("I will not overwrite previous results " "(unless you specify overwrite=True)") self.destination = destination def _clean_events_table(self): """Clean the events table. Remove duplicate events and sort the table by ext_timestamp. """ events = self.source enumerated_timestamps = list(enumerate(events.col('ext_timestamp'))) enumerated_timestamps.sort(key=operator.itemgetter(1)) unique_sorted_ids = self._find_unique_row_ids(enumerated_timestamps) new_events = self._replace_table_with_selected_rows( events, unique_sorted_ids) self.source = new_events self._normalize_event_ids(new_events) def _find_unique_row_ids(self, enumerated_timestamps): """Find the unique row_ids from enumerated timestamps.""" prev_timestamp = 0 unique_sorted_ids = [] for row_id, timestamp in enumerated_timestamps: if timestamp != prev_timestamp: # event is unique, so add it unique_sorted_ids.append(row_id) prev_timestamp = timestamp return unique_sorted_ids def _replace_table_with_selected_rows(self, table, row_ids): """Replace events table with selected rows. :param table: original table to be replaced. :param row_ids: row ids of the selected rows which should go in the destination table. """ tmptable = self.data.create_table(self.group, 't__events', description=table.description) selected_rows = table.read_coordinates(row_ids) tmptable.append(selected_rows) tmptable.flush() self.data.rename_node(tmptable, table.name, overwrite=True) return tmptable def _normalize_event_ids(self, events): """Normalize event ids. After sorting, the event ids no longer correspond to the row number. This can complicate finding the row id of a particular event. This method will replace the event_ids of all events by the row id. :param events: the events table to normalize. """ row_ids = list(range(len(events))) events.modify_column(column=row_ids, colname='event_id') def _create_results_table(self): """Create results table containing the events.""" self._tmp_events = self._create_empty_results_table() self._copy_events_into_table() def _create_empty_results_table(self): """Create empty results table with correct length.""" if self.limit: length = self.limit else: length = len(self.source) if '_t_events' in self.group: self.data.remove_node(self.group, '_t_events') table = self.data.create_table(self.group, '_t_events', self.processed_events_description, expectedrows=length) for _ in range(length): table.row.append() table.flush() return table def _copy_events_into_table(self): table = self._tmp_events source = self.source for col in pbar(source.colnames, show=self.progress): table.modify_column(stop=self.limit, colname=col, column=getattr(source.cols, col)[:self.limit]) table.flush() def _store_results_from_traces(self): table = self._tmp_events timings = self.process_traces() # Assign values to full table, column-wise. for idx in range(4): col = 't%d' % (idx + 1) table.modify_column(column=timings[:, idx], colname=col) table.flush() def process_traces(self): """Process traces to yield pulse timing information.""" if self.limit is not None: events = self.source.iterrows(stop=self.limit) else: events = self.source timings = self._process_traces_from_event_list(events, length=self.limit) return timings def _process_traces_from_event_list(self, events, length=None): """Process traces from a list of events. This is the method looping over all events. :param events: an iterable of the events :param length: an indication of the number of events, for use as a progress bar. Optional. """ result = [] for event in pbar(events, length=length, show=self.progress): timings = self._reconstruct_time_from_traces(event) result.append(timings) timings = np.array(result) return timings def _reconstruct_time_from_traces(self, event): """Reconstruct arrival times for a single event. This method loops over the traces. :param event: row from the events table. :return: arrival times in the detectors relative to trace start in ns. """ timings = [] for baseline, pulseheight, trace_idx in zip(event['baseline'], event['pulseheights'], event['traces']): if pulseheight < 0: # retain -1, -999 status flags in timing timings.append(pulseheight) elif pulseheight < ADC_THRESHOLD: timings.append(-999) else: trace = self._get_trace(trace_idx) timings.append( self._reconstruct_time_from_trace(trace, baseline)) timings = [ time * ADC_TIME_PER_SAMPLE if time not in ERR else time for time in timings ] return timings def _get_trace(self, idx): """Returns a trace given an index into the blobs array. Decompress a trace from the blobs array. :param idx: index into the blobs array :return: iterator over the pulseheight values """ blobs = self._get_blobs() try: trace = zlib.decompress(blobs[idx]).decode('utf-8').split(',') except zlib.error: trace = (zlib.decompress( blobs[idx][1:-1]).decode('utf-8').split(',')) if trace[-1] == '': del trace[-1] trace = (int(x) for x in trace) return trace def _get_blobs(self): return self.group.blobs def _reconstruct_time_from_trace(self, trace, baseline): """Reconstruct time of measurement from a trace. This method is doing the hard work. :param trace: array containing pulseheight values. :param baseline: baseline of the trace. :return: index in trace for arrival time of first particle. """ threshold = baseline + ADC_THRESHOLD value = self.first_above_threshold(trace, threshold) return value @staticmethod def first_above_threshold(trace, threshold): """Find the first element in the list equal or above threshold If no element matches the condition -999 will be returned. :param trace: iterable trace. :param threshold: value the trace has to be greater or equal to. :return: index in trace where a value is greater or equal to threshold. """ return next((i for i, x in enumerate(trace) if x >= threshold), -999) def _store_number_of_particles(self): """Store number of particles in the detectors. Process all pulseintegrals from the events and estimate the number of particles in each detector. """ table = self._tmp_events n_particles = self._process_pulseintegrals() for idx in range(4): col = 'n%d' % (idx + 1) table.modify_column(column=n_particles[:, idx], colname=col) table.flush() def _process_pulseintegrals(self): """Find MPVs using pulseintegrals to estimate number of particles :return: array with estimated number of particles per detector per event. """ n_particles = [] integrals = self.source.col('integrals') all_mpv = [] for detector_integrals in integrals.T: if (detector_integrals < 0).all(): all_mpv.append(np.nan) else: n, bins = np.histogram(detector_integrals, bins=np.linspace(0, 50000, 201)) find_mpv = FindMostProbableValueInSpectrum(n, bins) mpv, is_fitted = find_mpv.find_mpv() if is_fitted: all_mpv.append(mpv) else: all_mpv.append(np.nan) all_mpv = np.array(all_mpv) for event in self.source[:self.limit]: pulseintegrals = event['integrals'] # retain -1, -999 status flags pulseintegrals = np.where(pulseintegrals >= 0, pulseintegrals / all_mpv, pulseintegrals) # if mpv fit failed, value is nan. Make it -999 pulseintegrals = np.where(np.isnan(pulseintegrals), -999, pulseintegrals) n_particles.append(pulseintegrals) return np.array(n_particles) def _move_results_table_into_destination(self): if self.source.name == 'events': self.source.rename('_events') self.source = self.group._events if self.destination in self.group: self.data.remove_node(self.group, self.destination) self._tmp_events.rename(self.destination) def __repr__(self): if not self.data.isopen: return "<finished %s>" % self.__class__.__name__ else: return ("%s(%r, %r, source=%r, progress=%r)" % (self.__class__.__name__, self.data.filename, self.group._v_pathname, self.source._v_pathname, self.progress))
class PhenotypeValue(tables.IsDescription): """ Phenotype value class """ ecotype = tables.Int32Col() value = tables.Float32Col()
class Record(tables.IsDescription): col1 = tables.Int32Col() col2 = tables.Int32Col() col3 = tables.Float64Col() col4 = tables.Float64Col()
# read one file to extract TF and Electrod info TF = cf.Eph(AllEph[0]).TF Electrodes = cf.Eph(AllEph[0]).Electrodes AllData = H5.createEArray(DataGroup, 'All', tables.Float64Atom(), (TF * Electrodes, 0)) GFPData = H5.createEArray(DataGroup, 'GFP', tables.Float64Atom(), (TF, 0)) # Reading EphFile dans store into Tables with EArray for e in AllEph: dat = cf.Eph(e) AllData.append(dat.Data.reshape(np.array(dat.Data.shape).prod(), 1)) GFPData.append(dat.GFP.reshape(TF, 1)) ShapeOriginalData = H5.createArray('/', 'Shape', np.array(dat.Data.shape)) ModelParticle = { 'Name': tables.StringCol(40), 'Value': tables.Int32Col(shape=len(SubjectFactor)), 'Type': tables.StringCol(40) } InfoParticle = {} for c in InfoDict: InfoParticle[c] = tables.StringCol(256) AnovaAllParticle = { 'StatEffect': tables.StringCol(40), 'P': tables.Float64Col(shape=(TF, Electrodes)), 'F': tables.Float64Col(shape=(TF, Electrodes)) } AnovaGFPParticle = { 'StatEffect': tables.StringCol(40), 'P': tables.Float64Col(shape=(TF, 1)), 'F': tables.Float64Col(shape=(TF, 1))
class TriggerInfoData(tables.IsDescription): EventID = tables.Int64Col(pos=0) Sec = tables.Int32Col(pos=1) NanoSec = tables.Int32Col(pos=2)
def main(): # Argument parser parser = make_argparser() parser.add_argument( "--debug", action="store_true", help="Print debugging information", ) parser.add_argument( "--save_images", action="store_true", help="Save also all images", ) parser.add_argument( "--estimate_energy", type=str2bool, default=False, help="Estimate the events' energy with a regressor from\ protopipe.scripts.build_model", ) parser.add_argument("--regressor_dir", type=str, default="./", help="regressors directory") parser.add_argument( "--regressor_config", type=str, default=None, help="Configuration file used to produce regressor model") args = parser.parse_args() # Read configuration file cfg = load_config(args.config_file) try: # If the user didn't specify a site and/or and array... site = cfg["General"]["site"] array = cfg["General"]["array"] except KeyError: # ...raise an error and exit. print("\033[91m ERROR: make sure that both 'site' and 'array' are " "specified in the analysis configuration file! \033[0m") sys_exit(-1) if args.infile_list: filenamelist = [] for f in args.infile_list: filenamelist += glob("{}/{}".format(args.indir, f)) filenamelist.sort() else: raise ValueError("don't know which input to use...") if not filenamelist: print("no files found; check indir: {}".format(args.indir)) sys_exit(-1) else: print("found {} files".format(len(filenamelist))) # Get the IDs of the involved telescopes and associated cameras together # with the equivalent focal lengths from the first event allowed_tels, cams_and_foclens, subarray = prod3b_array( filenamelist[0], site, array) # keeping track of events and where they were rejected evt_cutflow = CutFlow("EventCutFlow") img_cutflow = CutFlow("ImageCutFlow") preper = EventPreparer( config=cfg, subarray=subarray, cams_and_foclens=cams_and_foclens, mode=args.mode, event_cutflow=evt_cutflow, image_cutflow=img_cutflow, ) # catch ctr-c signal to exit current loop and still display results signal_handler = SignalHandler() signal.signal(signal.SIGINT, signal_handler) # Regressor information regressor_method = cfg["EnergyRegressor"]["method_name"] try: estimation_weight = cfg["EnergyRegressor"]["estimation_weight"] except KeyError: estimation_weight = "STD" # wrapper for the scikit-learn regressor if args.estimate_energy is True: # Read configuration file regressor_config = load_config(args.regressor_config) log_10_target = regressor_config["Method"]["log_10_target"] regressor_files = (args.regressor_dir + "/regressor_{cam_id}_{regressor}.pkl.gz") reg_file = regressor_files.format( **{ "mode": args.mode, "wave_args": "mixed", # ToDo, control "regressor": regressor_method, "cam_id": "{cam_id}", }) regressors = load_models(reg_file, cam_id_list=cams_and_foclens.keys()) # COLUMN DESCRIPTOR AS DICTIONARY # Column descriptor for the file containing output training data.""" DataTrainingOutput = dict( # ====================================================================== # ARRAY obs_id=tb.Int16Col(dflt=1, pos=0), event_id=tb.Int32Col(dflt=1, pos=1), tel_id=tb.Int16Col(dflt=1, pos=2), N_LST=tb.Int16Col(dflt=1, pos=3), N_MST=tb.Int16Col(dflt=1, pos=4), N_SST=tb.Int16Col(dflt=1, pos=5), n_tel_reco=tb.FloatCol(dflt=1, pos=6), n_tel_discri=tb.FloatCol(dflt=1, pos=7), # ====================================================================== # DL1 hillas_intensity_reco=tb.Float32Col(dflt=1, pos=8), hillas_intensity=tb.Float32Col(dflt=1, pos=9), hillas_x_reco=tb.Float32Col(dflt=1, pos=10), hillas_y_reco=tb.Float32Col(dflt=1, pos=11), hillas_x=tb.Float32Col(dflt=1, pos=12), hillas_y=tb.Float32Col(dflt=1, pos=13), hillas_r_reco=tb.Float32Col(dflt=1, pos=14), hillas_r=tb.Float32Col(dflt=1, pos=15), hillas_phi_reco=tb.Float32Col(dflt=1, pos=16), hillas_phi=tb.Float32Col(dflt=1, pos=17), hillas_length_reco=tb.Float32Col(dflt=1, pos=18), hillas_length=tb.Float32Col(dflt=1, pos=19), hillas_width_reco=tb.Float32Col(dflt=1, pos=20), hillas_width=tb.Float32Col(dflt=1, pos=21), hillas_psi_reco=tb.Float32Col(dflt=1, pos=22), hillas_psi=tb.Float32Col(dflt=1, pos=23), hillas_skewness_reco=tb.Float32Col(dflt=1, pos=24), hillas_skewness=tb.Float32Col(dflt=1, pos=25), hillas_kurtosis=tb.Float32Col(dflt=1, pos=26), hillas_kurtosis_reco=tb.Float32Col(dflt=1, pos=27), leakage_intensity_width_1_reco=tb.Float32Col(dflt=np.nan, pos=28), leakage_intensity_width_2_reco=tb.Float32Col(dflt=np.nan, pos=29), leakage_intensity_width_1=tb.Float32Col(dflt=np.nan, pos=30), leakage_intensity_width_2=tb.Float32Col(dflt=np.nan, pos=31), concentration_cog=tb.Float32Col(dflt=np.nan, pos=32), concentration_core=tb.Float32Col(dflt=np.nan, pos=33), concentration_pixel=tb.Float32Col(dflt=np.nan, pos=34), # The following are missing from current ctapipe DL1 output # Not sure if it's worth to add them hillas_ellipticity_reco=tb.FloatCol(dflt=1, pos=35), hillas_ellipticity=tb.FloatCol(dflt=1, pos=36), max_signal_cam=tb.Float32Col(dflt=1, pos=37), pixels=tb.Int16Col(dflt=1, pos=38), clusters=tb.Int16Col(dflt=-1, pos=39), # ====================================================================== # DL2 - DIRECTION RECONSTRUCTION impact_dist=tb.Float32Col(dflt=1, pos=40), h_max=tb.Float32Col(dflt=1, pos=41), alt=tb.Float32Col(dflt=np.nan, pos=42), az=tb.Float32Col(dflt=np.nan, pos=43), err_est_pos=tb.Float32Col(dflt=1, pos=44), err_est_dir=tb.Float32Col(dflt=1, pos=45), xi=tb.Float32Col(dflt=np.nan, pos=46), offset=tb.Float32Col(dflt=np.nan, pos=47), mc_core_x=tb.FloatCol(dflt=1, pos=48), mc_core_y=tb.FloatCol(dflt=1, pos=49), reco_core_x=tb.FloatCol(dflt=1, pos=50), reco_core_y=tb.FloatCol(dflt=1, pos=51), mc_h_first_int=tb.FloatCol(dflt=1, pos=52), mc_x_max=tb.Float32Col(dflt=np.nan, pos=53), is_valid=tb.BoolCol(dflt=False, pos=54), good_image=tb.Int16Col(dflt=1, pos=55), # ====================================================================== # DL2 - ENERGY ESTIMATION true_energy=tb.FloatCol(dflt=1, pos=56), reco_energy=tb.FloatCol(dflt=np.nan, pos=57), reco_energy_tel=tb.Float32Col(dflt=np.nan, pos=58), # ====================================================================== # DL1 IMAGES # this is optional data saved by the user # since these data declarations require to know how many pixels # each saved image will have, # we add them later on, right before creating the table # We list them here for reference # true_image=tb.Float32Col(shape=(1855), pos=56), # reco_image=tb.Float32Col(shape=(1855), pos=57), # cleaning_mask_reco=tb.BoolCol(shape=(1855), pos=58), # not in ctapipe ) outfile = tb.open_file(args.outfile, mode="w") outTable = {} outData = {} for i, filename in enumerate(filenamelist): print("file: {} filename = {}".format(i, filename)) source = EventSource(input_url=filename, allowed_tels=allowed_tels, max_events=args.max_events) # loop that cleans and parametrises the images and performs the # reconstruction for each event for ( event, reco_image, cleaning_mask_reco, cleaning_mask_clusters, true_image, n_pixel_dict, hillas_dict, hillas_dict_reco, leakage_dict, concentration_dict, n_tels, max_signals, n_cluster_dict, reco_result, impact_dict, good_event, good_for_reco, ) in preper.prepare_event(source, save_images=args.save_images, debug=args.debug): if good_event: xi = angular_separation(event.simulation.shower.az, event.simulation.shower.alt, reco_result.az, reco_result.alt) offset = angular_separation( event.pointing.array_azimuth, event.pointing.array_altitude, reco_result.az, reco_result.alt, ) # Impact parameter reco_core_x = reco_result.core_x reco_core_y = reco_result.core_y # Height of shower maximum h_max = reco_result.h_max # Todo add conversion in number of radiation length, # need an atmosphere profile is_valid = True else: # something went wrong and the shower's reconstruction failed xi = np.nan * u.deg offset = np.nan * u.deg reco_core_x = np.nan * u.m reco_core_y = np.nan * u.m h_max = np.nan * u.m reco_result.alt = np.nan * u.deg reco_result.az = np.nan * u.deg is_valid = False reco_energy = np.nan reco_energy_tel = dict() # Not optimal at all, two loop on tel!!! # For energy estimation # Estimate energy only if the shower was reconstructed if (args.estimate_energy is True) and is_valid: weight_tel = np.zeros(len(hillas_dict.keys())) energy_tel = np.zeros(len(hillas_dict.keys())) for idx, tel_id in enumerate(hillas_dict.keys()): # use only images that survived cleaning and # parametrization if not good_for_reco[tel_id]: # bad images will get an undetermined energy # this is a per-telescope energy # NOT the estimated energy for the shower reco_energy_tel[tel_id] = np.nan continue cam_id = source.subarray.tel[tel_id].camera.camera_name moments = hillas_dict[tel_id] model = regressors[cam_id] ############################################################ # GET FEATURES ############################################################ # Read feature list from model configutation file features_basic = regressor_config["FeatureList"]["Basic"] features_derived = regressor_config["FeatureList"][ "Derived"] features = features_basic + list(features_derived) # Create a pandas Dataframe with basic quantities # This is needed in order to connect the I/O system of the # model inputs to the in-memory computation of this script data = pd.DataFrame({ "hillas_intensity": [moments.intensity], "hillas_width": [moments.width.to("deg").value], "hillas_length": [moments.length.to("deg").value], "hillas_x": [moments.x.to("deg").value], "hillas_y": [moments.y.to("deg").value], "hillas_phi": [moments.phi.to("deg").value], "hillas_r": [moments.r.to("deg").value], "leakage_intensity_width_1_reco": [leakage_dict[tel_id]['leak1_reco']], "leakage_intensity_width_2_reco": [leakage_dict[tel_id]['leak2_reco']], "leakage_intensity_width_1": [leakage_dict[tel_id]['leak1']], "leakage_intensity_width_2": [leakage_dict[tel_id]['leak2']], "concentration_cog": [concentration_dict[tel_id]['concentration_cog']], "concentration_core": [concentration_dict[tel_id]['concentration_core']], "concentration_pixel": [concentration_dict[tel_id]['concentration_pixel']], "az": [reco_result.az.to("deg").value], "alt": [reco_result.alt.to("deg").value], "h_max": [h_max.value], "impact_dist": [impact_dict[tel_id].to("m").value], }) # Compute derived features and add them to the dataframe for key, expression in features_derived.items(): data.eval(f'{key} = {expression}', inplace=True) # features_img = np.array( # [ # np.log10(moments.intensity), # np.log10(impact_dict[tel_id].value), # moments.width.value, # moments.length.value, # h_max.value, # ] # ) # sort features_to_use alphabetically to ensure order # preservation with model.fit in protopipe.mva features = sorted(features) # Select the values for the full set of features features_values = data[features].to_numpy() ############################################################ if estimation_weight == "STD": # Get an array of trees predictions_trees = np.array([ tree.predict(features_values) for tree in model.estimators_ ]) energy_tel[idx] = np.mean(predictions_trees, axis=0) weight_tel[idx] = np.std(predictions_trees, axis=0) else: data.eval(f'estimation_weight = {estimation_weight}', inplace=True) energy_tel[idx] = model.predict(features_values) weight_tel[idx] = data["estimation_weight"] if log_10_target: energy_tel[idx] = 10**energy_tel[idx] weight_tel[idx] = 10**weight_tel[idx] reco_energy_tel[tel_id] = energy_tel[idx] reco_energy = np.sum(weight_tel * energy_tel) / sum(weight_tel) else: for idx, tel_id in enumerate(hillas_dict.keys()): reco_energy_tel[tel_id] = np.nan for idx, tel_id in enumerate(hillas_dict.keys()): cam_id = source.subarray.tel[tel_id].camera.camera_name if cam_id not in outData: if args.save_images is True: # we define and save images content here, to make it # adaptive to different cameras n_pixels = source.subarray.tel[ tel_id].camera.geometry.n_pixels DataTrainingOutput["true_image"] = tb.Float32Col( shape=(n_pixels), pos=56) DataTrainingOutput["reco_image"] = tb.Float32Col( shape=(n_pixels), pos=57) DataTrainingOutput["cleaning_mask_reco"] = tb.BoolCol( shape=(n_pixels), pos=58) # not in ctapipe DataTrainingOutput[ "cleaning_mask_clusters"] = tb.BoolCol( shape=(n_pixels), pos=58) # not in ctapipe outTable[cam_id] = outfile.create_table( "/", cam_id, DataTrainingOutput, ) outData[cam_id] = outTable[cam_id].row moments = hillas_dict[tel_id] ellipticity = moments.width / moments.length # Write to file also the Hillas parameters that have been used # to calculate reco_results moments_reco = hillas_dict_reco[tel_id] ellipticity_reco = moments_reco.width / moments_reco.length outData[cam_id]["good_image"] = good_for_reco[tel_id] outData[cam_id]["is_valid"] = is_valid outData[cam_id]["impact_dist"] = impact_dict[tel_id].to( "m").value outData[cam_id]["max_signal_cam"] = max_signals[tel_id] outData[cam_id]["hillas_intensity"] = moments.intensity outData[cam_id]["N_LST"] = n_tels["LST_LST_LSTCam"] outData[cam_id]["N_MST"] = (n_tels["MST_MST_NectarCam"] + n_tels["MST_MST_FlashCam"] + n_tels["MST_SCT_SCTCam"]) outData[cam_id]["N_SST"] = (n_tels["SST_1M_DigiCam"] + n_tels["SST_ASTRI_ASTRICam"] + n_tels["SST_GCT_CHEC"]) outData[cam_id]["hillas_width"] = moments.width.to("deg").value outData[cam_id]["hillas_length"] = moments.length.to( "deg").value outData[cam_id]["hillas_psi"] = moments.psi.to("deg").value outData[cam_id]["hillas_skewness"] = moments.skewness outData[cam_id]["hillas_kurtosis"] = moments.kurtosis outData[cam_id]["h_max"] = h_max.to("m").value outData[cam_id]["err_est_pos"] = np.nan outData[cam_id]["err_est_dir"] = np.nan outData[cam_id][ "true_energy"] = event.simulation.shower.energy.to( "TeV").value outData[cam_id]["hillas_x"] = moments.x.to("deg").value outData[cam_id]["hillas_y"] = moments.y.to("deg").value outData[cam_id]["hillas_phi"] = moments.phi.to("deg").value outData[cam_id]["hillas_r"] = moments.r.to("deg").value outData[cam_id]["pixels"] = n_pixel_dict[tel_id] outData[cam_id]["obs_id"] = event.index.obs_id outData[cam_id]["event_id"] = event.index.event_id outData[cam_id]["tel_id"] = tel_id outData[cam_id]["xi"] = xi.to("deg").value outData[cam_id]["reco_energy"] = reco_energy outData[cam_id]["hillas_ellipticity"] = ellipticity.value outData[cam_id]["clusters"] = n_cluster_dict[tel_id] outData[cam_id]["n_tel_discri"] = n_tels["GOOD images"] outData[cam_id][ "mc_core_x"] = event.simulation.shower.core_x.to("m").value outData[cam_id][ "mc_core_y"] = event.simulation.shower.core_y.to("m").value outData[cam_id]["reco_core_x"] = reco_core_x.to("m").value outData[cam_id]["reco_core_y"] = reco_core_y.to("m").value outData[cam_id][ "mc_h_first_int"] = event.simulation.shower.h_first_int.to( "m").value outData[cam_id]["offset"] = offset.to("deg").value outData[cam_id][ "mc_x_max"] = event.simulation.shower.x_max.value # g / cm2 outData[cam_id]["alt"] = reco_result.alt.to("deg").value outData[cam_id]["az"] = reco_result.az.to("deg").value outData[cam_id]["reco_energy_tel"] = reco_energy_tel[tel_id] # Variables from hillas_dist_reco outData[cam_id]["n_tel_reco"] = n_tels["GOOD images"] outData[cam_id]["hillas_x_reco"] = moments_reco.x.to( "deg").value outData[cam_id]["hillas_y_reco"] = moments_reco.y.to( "deg").value outData[cam_id]["hillas_phi_reco"] = moments_reco.phi.to( "deg").value outData[cam_id][ "hillas_ellipticity_reco"] = ellipticity_reco.value outData[cam_id]["hillas_r_reco"] = moments_reco.r.to( "deg").value outData[cam_id]["hillas_skewness_reco"] = moments_reco.skewness outData[cam_id]["hillas_kurtosis_reco"] = moments_reco.kurtosis outData[cam_id]["hillas_width_reco"] = moments_reco.width.to( "deg").value outData[cam_id]["hillas_length_reco"] = moments_reco.length.to( "deg").value outData[cam_id]["hillas_psi_reco"] = moments_reco.psi.to( "deg").value outData[cam_id][ "hillas_intensity_reco"] = moments_reco.intensity outData[cam_id][ "leakage_intensity_width_1_reco"] = leakage_dict[tel_id][ "leak1_reco"] outData[cam_id][ "leakage_intensity_width_2_reco"] = leakage_dict[tel_id][ "leak2_reco"] outData[cam_id]["leakage_intensity_width_1"] = leakage_dict[ tel_id]["leak1"] outData[cam_id]["leakage_intensity_width_2"] = leakage_dict[ tel_id]["leak2"] outData[cam_id]["concentration_cog"] = concentration_dict[ tel_id]["concentration_cog"] outData[cam_id]["concentration_core"] = concentration_dict[ tel_id]["concentration_core"] outData[cam_id]["concentration_pixel"] = concentration_dict[ tel_id]["concentration_pixel"] # ======================= # IMAGES INFORMATION # ======================= if args.save_images is True: # we define and save images content here, to make it # adaptive to different cameras outData[cam_id]["true_image"] = true_image[tel_id] outData[cam_id]["reco_image"] = reco_image[tel_id] outData[cam_id]["cleaning_mask_reco"] = cleaning_mask_reco[ tel_id] outData[cam_id][ "cleaning_mask_clusters"] = cleaning_mask_clusters[ tel_id] # ======================= outData[cam_id].append() if signal_handler.stop: break if signal_handler.stop: break # make sure that all the events are properly stored for table in outTable.values(): table.flush() print(bcolors.BOLD + "\n\n==================================================\n" + "Statistical summary of processed events and images\n" + "==================================================\n" # + bcolors.ENDC ) evt_cutflow() # Catch specific cases triggered_events = evt_cutflow.cuts["min2Tels trig"][1] reconstructed_events = evt_cutflow.cuts["min2Tels reco"][1] if triggered_events == 0: print("\033[93mWARNING: No events have been triggered" " by the selected telescopes! \033[0m") else: print("\n") img_cutflow() if reconstructed_events == 0: print("\033[93m WARNING: None of the triggered events have been " "properly reconstructed by the selected telescopes!\n" "DL1 file will be empty! \033[0m") print(bcolors.ENDC)
class Meta(tb.IsDescription): elapsed_day = tb.Int32Col() observed_day = tb.Int32Col() # MJD band = tb.Int8Col() index = tb.Int8Col()
import sys from time import perf_counter as clock from time import process_time as cpuclock import numpy as np import tables as tb class Test(tb.IsDescription): ngroup = tb.Int32Col(pos=1) ntable = tb.Int32Col(pos=2) nrow = tb.Int32Col(pos=3) #string = StringCol(itemsize=500, pos=4) TestDict = { "ngroup": tb.Int32Col(pos=1), "ntable": tb.Int32Col(pos=2), "nrow": tb.Int32Col(pos=3), } def createFileArr(filename, ngroups, ntables, nrows): # First, create the groups # Open a file in "w"rite mode fileh = tb.open_file(filename, mode="w", title="PyTables Stress Test") for k in range(ngroups): # Create the group fileh.create_group("/", 'group%04d' % k, "Group %d" % k)
class Meta(tb.IsDescription): observed_day = tb.Int32Col() # MJD band = tb.Int8Col()
def _type_to_col(t, pos): if t is int: return tables.Int32Col(pos=pos) if t is float: return tables.Float32Col(pos=pos) raise NotImplementedError(t)
class Nafc_Gap_Laser(Nafc_Gap): PARAMS = copy(Nafc_Gap.PARAMS) PARAMS['laser_probability'] = { 'tag': 'Probability (of trials whose targets match laser_mode) of laser being turned on (0-1)', 'type': 'float' } PARAMS['laser_mode'] = { 'tag': 'Laser Mode', 'type': 'list', 'values': { 'L': 0, 'R': 1, 'Both': 2 } } PARAMS['laser_freq'] = { 'tag': 'Laser Pulse Frequency (Hz)', 'type': 'float' } PARAMS['laser_duty_cycle'] = { 'tag': 'Laser Duty Cycle (0-1)', 'type': 'float' } PARAMS['laser_durations'] = { 'tag': 'Laser durations (ms), list-like [10, 20]. if blank, use durations from stimuli', 'type': 'str' } HARDWARE = copy(Nafc_Gap.HARDWARE) HARDWARE['LASERS'] = {'L': gpio.Digital_Out, 'R': gpio.Digital_Out} HARDWARE['LEDS']['TOP'] = gpio.Digital_Out TrialData = copy(Nafc_Gap.TrialData) TrialData.laser = tables.Int32Col() TrialData.laser_duration = tables.Float32Col() def __init__(self, laser_probability: float, laser_mode: str, laser_freq: float, laser_duty_cycle: float, laser_durations: typing.Union[str, list], **kwargs): """ Gap detection task with ability to control lasers via TTL logic for optogenetics .. note:: Subclasses like these will be made obsolete with the completion of stimulus managers Args: laser_probability (float): laser_mode: laser_freq: laser_duty_cycle: laser_durations: """ self.laser_probability = float(laser_probability) self.laser_mode = laser_mode self.laser_freq = float(laser_freq) self.laser_duty_cycle = float(laser_duty_cycle) self.laser_durations = laser_durations super(Nafc_Gap_Laser, self).__init__(**kwargs) # -------------------------------------- # create description of laser pulses # make a pair of lists, values (on/off) and durations (ms) # use them to create pigpio scripts using the Digital_Out.store_series() method # get the durations of on and off for a single cycle cycle_duration = (1 / self.laser_freq) * 1000 duty_cycle_on = self.laser_duty_cycle * cycle_duration duty_cycle_off = cycle_duration - duty_cycle_on self.duration_ids = [] if isinstance(self.laser_durations, list): # iterate through durations and create lists for each for duration in self.laser_durations: # get number of repeats to make n_cycles = np.floor(duration / cycle_duration) durations = [duty_cycle_on, duty_cycle_off] * n_cycles values = [1, 0] * n_cycles # pad any incomplete cycles dur_remaining = duration - (cycle_duration * n_cycles) if dur_remaining < duty_cycle_on: durations.append(dur_remaining) values.append(1) else: durations.extend( [duty_cycle_on, dur_remaining - duty_cycle_on]) values.extend([1, 0]) # store pulses as pigpio scripts self.hardware['LASERS']['L'].store_series(duration, values=values, durations=durations) self.hardware['LASERS']['R'].store_series(duration, values=values, durations=durations) else: # use the durations of the stimuli self.logger.exception( 'reading durations from stimulus manager not implemented') raise NotImplementedError( 'read durations from the stimulus manager') # ----------------------------------- # create a pulse for the LED that's equal to the longest stimulus duration # use find_recursive to find all durations # FIXME: implement stimulus managers properly, including API to get attributes of stimuli stim_durations = list(find_recursive('duration', kwargs['stim'])) max_duration = np.max(stim_durations) self.hardware['LEDS']['TOP'].store_series('on', values=1, durations=max_duration) def request(self, *args, **kwargs): # call the super method data = super(Nafc_Gap_Laser, self).request(*args, **kwargs) # handle laser logic # if the laser_mode is fulfilled, roll for a laser test_laser = False if self.laser_mode == "L" and self.target == "L": test_laser = True elif self.laser_mode == "R" and self.target == "R": test_laser = True elif self.laser_mode == "Both": test_laser = True duration = 0 do_laser = False if test_laser: # if we've rolled correctly for a laser... if np.random.rand() <= self.laser_probability: do_laser = True # pick a random duration duration = np.random.choice(self.laser_durations) # insert the laser triggers before the rest of the triggers self.triggers['C'].insert( 0, lambda: self.hardware['LASERS']['L'].series(id=duration)) self.triggers['C'].insert( 0, lambda: self.hardware['LASERS']['R'].series(id=duration)) # always turn the light on self.triggers['C'].insert( 0, lambda: self.hardware['LEDS']['TOP'].series(id='on')) # store the data about the laser status data['laser'] = do_laser data['laser_duration'] = duration # return the data created by the original task return data
class Particle(tb.IsDescription): name = tb.StringCol(16, pos=1) # 16-character String lati = tb.Int32Col(pos=2) # integer longi = tb.Int32Col(pos=3) # integer vector = tb.Int32Col(shape=(2,), pos=4) # Integer matrix2D = tb.Float64Col(shape=(2, 2), pos=5) # double (double-precision)
class DescriptionForTradesTable(tables.IsDescription): price = tables.Float32Col() amount = tables.Float32Col() timestamp = tables.Float32Col() tid = tables.Int32Col() type = tables.StringCol(3)