def StackCalSoln_Description(nPos=1): strLength = 100 description = { "intTime" : tables.UInt16Col(), # integration time used for each dither position "nPos" : tables.UInt16Col(), # number of dither positions "startTimes" : tables.UInt32Col(nPos), # list of data start times for each position "stopTimes" : tables.UInt32Col(nPos), # physical y location "darkSpan" : tables.UInt32Col(2), # start and stop time for dark data "flatSpan" : tables.UInt32Col(2), # start and stop time for flat data "xPos" : tables.UInt16Col(nPos), # rough guess at X centroid for each position "yPos" : tables.UInt16Col(nPos), # rough guess at Y centroid for each position "numRows" : tables.UInt16Col(), # y-dimension of image "numCols" : tables.UInt16Col(), # x-dimension of image "upSample" : tables.UInt16Col(), # divide each pixel into upSample^2 for subpix registration "padFraction" : tables.Float64Col(), # x-dimension of image "coldCut" : tables.UInt8Col(), # any pixels with counts<coldCut is set to NAN during regis. "fitPos" : tables.BoolCol(), # boolean flag to perform fitting for registration "target" : tables.StringCol(strLength), # name of target object "date" : tables.StringCol(strLength), # date of observation "imgDir" : tables.StringCol(strLength), # location of .IMG files "binDir" : tables.StringCol(strLength), # location of .bin files "outputDir" : tables.StringCol(strLength), # location where stack was originally saved "fitsPath" : tables.StringCol(strLength), # full path (dir+filename) for FITS file with stacked image "refFile" : tables.StringCol(strLength), # path to reference file for image registration "useImg" : tables.BoolCol(), # boolean flag to use .IMG or .bin data files in stacking "doHPM" : tables.StringCol(strLength), # string controlling manner that hot pix masks are used "subtractDark" : tables.BoolCol(), # boolean flag to apply dark "divideFlat" : tables.BoolCol(), # boolean flag to apply flat "apMaskRadPrim" : tables.UInt16Col(), # radius of aperture mask around primary object "apMaskRadSec" : tables.UInt16Col()} # radius of aperture mask around secondary return description
class _Flavors(tables.IsDescription): """Storage of neutrino flavor data. """ nu_e = tables.BoolCol(dflt=False) nu_e_bar = tables.BoolCol(dflt=False) nu_x = tables.BoolCol(dflt=False) nu_x_bar = tables.BoolCol(dflt=False)
class _Hierarchy(tables.IsDescription): """Storage of neutrino mass hierarchy. """ any = tables.BoolCol(dflt=False) none = tables.BoolCol(dflt=False) normal = tables.BoolCol(dflt=False) inverted = tables.BoolCol(dflt=False)
class HMMInfoParticle(tables.IsDescription): # HMM_ID, taste, din_channel, n_cells, time_start, time_end, thresh, # unit_type, n_repeats, dt, n_states, n_iters, BIC, cost, converged, # area hmm_id = tables.Int16Col() taste = tables.StringCol(45) channel = tables.Int16Col() n_cells = tables.Int32Col() unit_type = tables.StringCol(15) n_trials = tables.Int32Col() dt = tables.Float64Col() max_iter = tables.Int32Col() threshold = tables.Float64Col() time_start = tables.Int32Col() time_end = tables.Int32Col() n_repeats = tables.Int16Col() n_states = tables.Int32Col() n_iterations = tables.Int32Col() BIC = tables.Float64Col() cost = tables.Float64Col() converged = tables.BoolCol() fitted = tables.BoolCol() max_log_prob = tables.Float64Col() log_likelihood = tables.Float64Col() area = tables.StringCol(15) hmm_class = tables.StringCol(20) notes = tables.StringCol(40)
def store_coincidences(self, station_numbers=None): """Store the previously found coincidences. After having searched for coincidences, you can store the more user-friendly results in the ``coincidences`` group using this method. It also created a ``c_index`` and ``s_index`` table to find the source events. :param station_numbers: optional list of station_numbers. If given these will be used to attach correct numbers to the station column names in the coincidences table. Otherwise they will simply be numbered by id. This list must be the same length as the station_groups. """ n_coincidences = len(self._src_c_index) if station_numbers is not None: if len(station_numbers) != len(self.station_groups): raise RuntimeError( "Number of station numbers must equal number of groups.") self.station_numbers = station_numbers s_columns = { 's%d' % number: tables.BoolCol(pos=p) for p, number in enumerate(station_numbers, 12) } else: self.station_numbers = None s_columns = { 's%d' % n: tables.BoolCol(pos=(n + 12)) for n, _ in enumerate(self.station_groups) } description = storage.Coincidence description.columns.update(s_columns) self.coincidences = self.data.create_table(self.coincidence_group, 'coincidences', description, expectedrows=n_coincidences) self.c_index = [] for coincidence in pbar(self._src_c_index, show=self.progress): self._store_coincidence(coincidence) c_index = self.data.create_vlarray(self.coincidence_group, 'c_index', tables.UInt32Col(shape=2), expectedrows=n_coincidences) for observables_idx in pbar(self.c_index, show=self.progress): c_index.append(observables_idx) c_index.flush() s_index = self.data.create_vlarray(self.coincidence_group, 's_index', tables.VLStringAtom(), expectedrows=len( self.station_groups)) for station_group in self.station_groups: s_index.append(station_group.encode('utf-8')) s_index.flush()
class fraud_data(t.IsDescription): transactionIndex = t.UInt16Col() #points twords the transaction data isFraud = t.BoolCol() fraudScore = t.Float64Col() dateEvaluated = t.TimeCol() examinedByOperator = t.BoolCol() ownerContacted = t.BoolCol()
class AnonHMMInfoParticle(tables.IsDescription): # HMM_ID, taste, din_channel, n_cells, time_start, time_end, thresh, # unit_type, n_repeats, dt, n_states, n_iters, BIC, cost, converged, # area # info particle for anonymous hmm data, so if hdf5 store isn't tied to a # single recording, adds column for rec_dir hmm_id = tables.Int16Col() taste = tables.StringCol(45) channel = tables.Int32Col() n_cells = tables.Int32Col() unit_type = tables.StringCol(15) n_trials = tables.Int32Col() dt = tables.Float64Col() max_iter = tables.Int32Col() threshold = tables.Float64Col() time_start = tables.Int32Col() time_end = tables.Int32Col() n_repeats = tables.Int16Col() n_states = tables.Int32Col() n_iterations = tables.Int32Col() BIC = tables.Float64Col() cost = tables.Float64Col() converged = tables.BoolCol() fitted = tables.BoolCol() max_log_prob = tables.Float64Col() log_likelihood = tables.Float64Col() area = tables.StringCol(15) hmm_class = tables.StringCol(20) notes = tables.StringCol(30) rec_dir = tables.StringCol(150)
class RecoEvent(tb.IsDescription): obs_id = tb.Int16Col(dflt=-1, pos=0) event_id = tb.Int32Col(dflt=-1, pos=1) NTels_trig = tb.Int16Col(dflt=0, pos=2) NTels_reco = tb.Int16Col(dflt=0, pos=3) NTels_reco_lst = tb.Int16Col(dflt=0, pos=4) NTels_reco_mst = tb.Int16Col(dflt=0, pos=5) NTels_reco_sst = tb.Int16Col(dflt=0, pos=6) pointing_az = tb.Float32Col(dflt=np.nan, pos=7) pointing_alt = tb.Float32Col(dflt=np.nan, pos=8) true_az = tb.Float32Col(dflt=np.nan, pos=9) true_alt = tb.Float32Col(dflt=np.nan, pos=10) true_energy = tb.Float32Col(dflt=np.nan, pos=11) reco_energy = tb.Float32Col(dflt=np.nan, pos=12) reco_alt = tb.Float32Col(dflt=np.nan, pos=13) reco_az = tb.Float32Col(dflt=np.nan, pos=14) offset = tb.Float32Col(dflt=np.nan, pos=15) xi = tb.Float32Col(dflt=np.nan, pos=16) ErrEstPos = tb.Float32Col(dflt=np.nan, pos=17) ErrEstDir = tb.Float32Col(dflt=np.nan, pos=18) gammaness = tb.Float32Col(dflt=np.nan, pos=19) success = tb.BoolCol(dflt=False, pos=20) score = tb.Float32Col(dflt=np.nan, pos=21) h_max = tb.Float32Col(dflt=np.nan, pos=22) reco_core_x = tb.Float32Col(dflt=np.nan, pos=23) reco_core_y = tb.Float32Col(dflt=np.nan, pos=24) true_core_x = tb.Float32Col(dflt=np.nan, pos=25) true_core_y = tb.Float32Col(dflt=np.nan, pos=26) is_valid = tb.BoolCol(dflt=False, pos=27)
class MemoryRegionInfo(tables.IsDescription): short_name = tables.StringCol(255) parent_name = tables.StringCol(255) name = tables.StringCol(255) comments = tables.StringCol(512) include_children = tables.BoolCol() reclassifiable = tables.BoolCol() do_print = tables.BoolCol()
class LightningStatus(tables.IsDescription): event_id = tables.UInt32Col(pos=0) timestamp = tables.Time32Col(pos=1) close_rate = tables.Int16Col(pos=2) total_rate = tables.Int16Col(pos=3) close_alarm = tables.BoolCol(pos=4) sever_alarm = tables.BoolCol(pos=5) current_heading = tables.Float32Col(pos=6)
class digital_mapping_particle(tables.IsDescription): '''Pytables particle for storing digital input/output mappings ''' channel = tables.Int16Col() name = tables.StringCol(20) palatability_rank = tables.Int16Col() laser = tables.BoolCol() spike_array = tables.BoolCol() exclude = tables.BoolCol() laser_channels = tables.BoolCol()
class SkipEntry(tables.IsDescription): pc = tables.UInt64Col() pclo = tables.UInt32Col() pchi = tables.UInt32Col() disasm = tables.StringCol(256) thumb = tables.BoolCol() resumepc = tables.UInt64Col() resumepclo = tables.UInt32Col() resumepchi = tables.UInt32Col() isfunction = tables.BoolCol()
class Trades(tables.IsDescription): time = tables.Int64Col() trader_id = tables.Int64Col() trade_id = tables.Int64Col() sequence_id = tables.Int64Col() side = tables.Int8Col() price = tables.Float64Col() quantity = tables.Int64Col() origin_id = tables.Int8Col() is_auction = tables.BoolCol() is_aggressor = tables.BoolCol()
class WriteEntry(tables.IsDescription): pc = tables.UInt64Col() pclo = tables.UInt32Col() pchi = tables.UInt32Col() thumb = tables.BoolCol() reg0 = tables.StringCol(4) reg1 = tables.StringCol(4) reg2 = tables.StringCol(4) reg3 = tables.StringCol(4) reg4 = tables.StringCol(4) writesize = tables.Int64Col() halt = tables.BoolCol() # whether to insert a breakpoint here
def store_coincidences(self, cluster=None): """Store the previously found coincidences. After having searched for coincidences, you can store the more user-friendly results in the ``coincidences`` group using this method. It also created a ``c_index`` and ``s_index`` table to find the source events. """ n_coincidences = len(self._src_c_index) if cluster: self.cluster = cluster self.coincidence_group._v_attrs.cluster = cluster s_columns = { 's%d' % station.number: tables.BoolCol(pos=p) for p, station in enumerate(cluster.stations, 12) } else: self.cluster = None s_columns = { 's%d' % n: tables.BoolCol(pos=(n + 12)) for n, _ in enumerate(self.station_groups) } description = storage.Coincidence description.columns.update(s_columns) self.coincidences = self.data.create_table(self.coincidence_group, 'coincidences', description, expectedrows=n_coincidences) self.c_index = [] for coincidence in pbar(self._src_c_index, show=self.progress): self._store_coincidence(coincidence) c_index = self.data.create_vlarray(self.coincidence_group, 'c_index', tables.UInt32Col(shape=2), expectedrows=n_coincidences) for observables_idx in pbar(self.c_index, show=self.progress): c_index.append(observables_idx) c_index.flush() s_index = self.data.create_vlarray(self.coincidence_group, 's_index', tables.VLStringAtom(), expectedrows=len( self.station_groups)) for station_group in self.station_groups: s_index.append(station_group) s_index.flush()
class CustomRecord(tables.IsDescription): class data(tables.IsDescription): velocity = tables.Float64Col(pos=1, shape=3) density = tables.Float64Col(pos=2) mask = tables.BoolCol(pos=1, shape=(3, ))
def _create_reconstruction_group_and_tables(self, results_group, overwrite): if results_group in self.data: if overwrite: self.data.remove_node(results_group, recursive=True) else: raise RuntimeError( "Result group exists, but overwrite is False") head, tail = os.path.split(results_group) group = self.data.create_group(head, tail) stations_description = { 's%d' % u: tables.BoolCol() for u in self.stations } description = self.reconstruction_description description.update(stations_description) self.reconstruction = self.data.create_table(group, 'reconstructions', description) description = self.reconstruction_coincidence_description description.update(stations_description) self.reconstruction_coincidences = \ self.data.create_table(group, 'coincidences', description) return group
class Spectrum_12_4096(tb.IsDescription): """ PyTables table descriptor: storage of spectral data For hipsr_200_16384 firmware: 200 MHz, 16384 channels """ id = tb.Int32Col(pos=0) # Unique ID timestamp = tb.Time64Col(pos=1) # Timestamp (at BRAM read) xx = tb.UInt32Col(shape=4096, pos=2) # XX Autocorrelation data yy = tb.UInt32Col(shape=4096, pos=3) # YY Autocorrelation data # re_xy = tb.Int32Col(shape=16384,pos=4) # XY Cross correlation - real #im_xy = tb.Int32Col(shape=16384,pos=5) # XY Cross correlation - imag xx_cal_on = tb.Int32Col(shape=16, pos=6) # Noise diode ON, X pol xx_cal_off = tb.Int32Col(shape=16, pos=7) # Noise diode OFF, X pol yy_cal_on = tb.Int32Col(shape=16, pos=8) # Noise diode ON, Y pol yy_cal_off = tb.Int32Col(shape=16, pos=9) # Noise diode OFF, Y pol fft_of = tb.BoolCol(pos=10) # FFT overflow flag adc_clip = tb.BoolCol(pos=11) # ADC clipping flag
class LadderTableSchema(tables.IsDescription): epsilon = tables.Float32Col() num_leapfrog_steps = tables.Int32Col() beta = tables.Float32Col() ladder_idx = tables.Int32Col() distr_hash = tables.Float32Col() mjhmc = tables.BoolCol()
def _prepare_coincidence_tables(self): """Create coincidence tables These are the same as the tables created by :class:`~sapphire.analysis.coincidences.CoincidencesESD`. This makes it easy to link events detected by multiple stations. """ self.coincidence_group = self.data.create_group(self.output_path, 'coincidences', createparents=True) try: self.coincidence_group._v_attrs.cluster = self.cluster except tables.HDF5ExtError: warnings.warn('Unable to store cluster object, to large for HDF.') description = storage.Coincidence s_columns = { 's%d' % station.number: tables.BoolCol(pos=p) for p, station in enumerate(self.cluster.stations, 12) } description.columns.update(s_columns) self.coincidences = self.data.create_table(self.coincidence_group, 'coincidences', description) self.c_index = self.data.create_vlarray(self.coincidence_group, 'c_index', tables.UInt32Col(shape=2)) self.s_index = self.data.create_vlarray(self.coincidence_group, 's_index', tables.VLStringAtom())
def test_make_tabledef(empty_dataframe): tabledef = _make_tabledef(empty_dataframe.to_records(index=False).dtype, 32) expected_tabledef = {'int_value' : tb.Int32Col ( shape=(), dflt=0 , pos=0), 'float_value' : tb.Float32Col( shape=(), dflt=0 , pos=1), 'bool_value' : tb.BoolCol ( shape=(), dflt=False, pos=2), 'str_value' : tb.StringCol (itemsize=32, shape=(), dflt=b'' , pos=3)} assert tabledef == expected_tabledef
class Record(tb.IsDescription): var1 = tb.StringCol(itemsize=4, dflt=b"abcd", pos=0) var2 = tb.StringCol(itemsize=1, dflt=b"a", pos=1) var3 = tb.BoolCol(dflt=1) var4 = tb.Int8Col(dflt=1) var5 = tb.UInt8Col(dflt=1) var6 = tb.Int16Col(dflt=1) var7 = tb.UInt16Col(dflt=1) var8 = tb.Int32Col(dflt=1) var9 = tb.UInt32Col(dflt=1) var10 = tb.Int64Col(dflt=1) var11 = tb.Float32Col(dflt=1.0) var12 = tb.Float64Col(dflt=1.0) var13 = tb.ComplexCol(itemsize=8, dflt=(1.+0.j)) var14 = tb.ComplexCol(itemsize=16, dflt=(1.+0.j)) if hasattr(tb, 'Float16Col'): var15 = tb.Float16Col(dflt=1.0) if hasattr(tb, 'Float96Col'): var16 = tb.Float96Col(dflt=1.0) if hasattr(tb, 'Float128Col'): var17 = tb.Float128Col(dflt=1.0) if hasattr(tb, 'Complex196Col'): var18 = tb.ComplexCol(itemsize=24, dflt=(1.+0.j)) if hasattr(tb, 'Complex256Col'): var19 = tb.ComplexCol(itemsize=32, dflt=(1.+0.j))
def _create_coincidences_tables(file, group, station_groups): """Setup coincidence tables :return: the created coincidences group. """ coin_group = group + '/coincidences' # Create coincidences table description = storage.Coincidence s_columns = { 's%d' % station: tables.BoolCol(pos=p) for p, station in enumerate(station_groups, 12) } description.columns.update(s_columns) coincidences = file.create_table(coin_group, 'coincidences', description, createparents=True) # Create c_index file.create_vlarray(coin_group, 'c_index', tables.UInt32Col(shape=2)) # Create and fill s_index s_index = file.create_vlarray(coin_group, 's_index', tables.VLStringAtom()) for station_group in itervalues(station_groups): s_index.append(station_group['group'].encode('utf-8')) return coincidences._v_parent
def prepare_output(self): """Prepare output table""" dest_path = os.path.join(self.dest_group, self.destination) if dest_path in self.dest_data: if self.overwrite: self.dest_data.remove_node(dest_path, recursive=True) else: raise RuntimeError("Reconstructions table already exists for " "%s, and overwrite is False" % self.dest_group) s_columns = { 's%d' % station.number: tables.BoolCol(pos=p) for p, station in enumerate(self.cluster.stations, 26) } description = ReconstructedCoincidence description.columns.update(s_columns) self.reconstructions = self.dest_data.create_table( self.dest_group, self.destination, description, expectedrows=self.coincidences.nrows, createparents=True) try: self.reconstructions._v_attrs.cluster = self.cluster except tables.HDF5ExtError: warnings.warn('Unable to store cluster object, to large for HDF.')
class LongWrites(tables.IsDescription): breakaddr = tables.UInt64Col() # where write loop starts breakaddrlo = tables.UInt32Col() # where write loop starts breakaddrhi = tables.UInt32Col() # where write loop starts writeaddr = tables.UInt64Col() # where write loop starts writeaddrlo = tables.UInt32Col() # where write loop starts writeaddrhi = tables.UInt32Col() # where write loop starts contaddr = tables.UInt64Col() # pc after loop thumb = tables.BoolCol() # if write is at thumb address inplace = tables.BoolCol() writesize = tables.UInt64Col() start = tables.UInt64Col() startlo = tables.UInt32Col() starthi = tables.UInt32Col() end = tables.UInt64Col() endlo = tables.UInt32Col() endhi = tables.UInt32Col()
class LightningConfig(tables.IsDescription): event_id = tables.UInt32Col(pos=0) timestamp = tables.Time32Col(pos=1) com_port = tables.UInt8Col(pos=2) baud_rate = tables.Int16Col(pos=3) station_id = tables.UInt32Col(pos=4) database_name = tables.Int32Col(dflt=-1, pos=5) help_url = tables.Int32Col(dflt=-1, pos=6) daq_mode = tables.BoolCol(pos=7) latitude = tables.Float64Col(pos=8) longitude = tables.Float64Col(pos=9) altitude = tables.Float64Col(pos=10) squelch_seting = tables.Int32Col(pos=11) close_alarm_distance = tables.Int32Col(pos=12) severe_alarm_distance = tables.Int32Col(pos=13) noise_beep = tables.BoolCol(pos=14) minimum_gps_speed = tables.Int32Col(pos=15) angle_correction = tables.Float32Col(pos=16)
class ReconstructedEvent(ReconstructedCoincidence): """Store information about reconstructed events .. attribute:: id Index referring to the id of the event that was reconstructed. .. attribute:: d1,d2,d3,d4 Booleans indicating which detectors participated in the reconstruction. """ d1 = tables.BoolCol(pos=22) d2 = tables.BoolCol(pos=23) d3 = tables.BoolCol(pos=24) d4 = tables.BoolCol(pos=25)
class FramaCDstEntry(tables.IsDescription): line = tables.StringCol(512) # file/lineno lvalue = tables.StringCol(512) # lvalue as reported by framac dstlo = tables.UInt32Col() # low value of write dst range dsthi = tables.UInt32Col() # high value of write dst range dst_not_in_ram = tables.BoolCol() # true if range is not RAM writepc = tables.UInt32Col() # corresponding store instruction PC to src line (if just 1) origpc = tables.UInt32Col() # corresponding store instruction PC to src line (if just 1) substage = tables.UInt8Col()
class Search(tb.IsDescription): nrows = tb.Int32Col(pos=0) rowsel = tb.Int32Col(pos=1) time1 = tb.Float64Col(pos=2) time2 = tb.Float64Col(pos=3) tcpu1 = tb.Float64Col(pos=4) tcpu2 = tb.Float64Col(pos=5) rowsec1 = tb.Float64Col(pos=6) rowsec2 = tb.Float64Col(pos=7) psyco = tb.BoolCol(pos=8)
class HDF5Frame(tables.IsDescription): timestamp = tables.UInt64Col() # Microseconds sentinel_start = tables.UInt8Col() sender_timestamp = tables.UInt32Col() sequence = tables.UInt16Col() id = tables.UInt16Col() rtr = tables.BoolCol() length = tables.UInt8Col() data = tables.UInt8Col(shape=(8, )) sentinel_end = tables.UInt8Col()