def create_input_group(h5f, title="input data at transmitter", rolloff_dflt=np.nan, attrs={}, arrays=["symbols", "bits"], **kwargs): """ Create the table for saving the input symbols and bits Parameters ---------- h5f : string or h5filehandle The file to use, if a string create or open new file title: string, optional The title description of the group attrs: dict, optional attributes on the table arrays: list, optional name of arrays referenced in the table **kwargs: keyword arguments passed to create_table/array, it is highly recommended to set expectedrows Returns ------- h5f : h5filehandle Pytables handle to the hdf file """ try: gr = h5f.create_group("/", "input", title=title) except AttributeError: h5f = tb.open_file(h5f, "a") gr = h5f.create_group("/", "input", title=title) # if no shape for input syms or bits is given use scalar t_in = h5f.create_table(gr, "signal", { "id": tb.Int64Col(), "idx_symbols": tb.Int64Col(dflt=0), "idx_bits": tb.Int64Col(dflt=0), "rolloff": tb.Float64Col(dflt=rolloff_dflt) }, title="parameters of input signal", **kwargs) setattr(t_in.attrs, "arrays", arrays) arr_syms = h5f.create_mdvlarray(gr, "symbols", tb.ComplexAtom(itemsize=16, dflt=np.nan), title="sent symbols", **kwargs) arr_bits = h5f.create_mdvlarray(gr, "bits", tb.BoolAtom(), title="sent bits", **kwargs) for k, v in attrs: setattr(t_in.attrs, k, v) return h5f
class ResultRecord(tables.IsDescription): """ Kruskal Wallis """ chromosome = tables.Int32Col() position = tables.Int32Col() score = tables.Float64Col() maf = tables.Float32Col() mac = tables.Int32Col()
class info(tb.IsDescription): _v_pos = 1 name = tb.StringCol(10) value = tb.Float64Col(pos=0) y2 = tb.Float64Col(dflt=1, shape=(2, 3), pos=1) z2 = tb.UInt8Col(dflt=1) class info2(tb.IsDescription): y3 = tb.Float64Col(dflt=1, shape=(2, 3)) z3 = tb.UInt8Col(dflt=1) name = tb.StringCol(10) value = tb.EnumCol(colors, 'blue', base='uint32', shape=(1,)) class info3(tb.IsDescription): name = tb.StringCol(10) value = tb.Time64Col() y4 = tb.Float64Col(dflt=1, shape=(2, 3)) z4 = tb.UInt8Col(dflt=1)
class LightningConfig(tables.IsDescription): event_id = tables.UInt32Col(pos=0) timestamp = tables.Time32Col(pos=1) com_port = tables.UInt8Col(pos=2) baud_rate = tables.Int16Col(pos=3) station_id = tables.UInt32Col(pos=4) database_name = tables.Int32Col(dflt=-1, pos=5) help_url = tables.Int32Col(dflt=-1, pos=6) daq_mode = tables.BoolCol(pos=7) latitude = tables.Float64Col(pos=8) longitude = tables.Float64Col(pos=9) altitude = tables.Float64Col(pos=10) squelch_seting = tables.Int32Col(pos=11) close_alarm_distance = tables.Int32Col(pos=12) severe_alarm_distance = tables.Int32Col(pos=13) noise_beep = tables.BoolCol(pos=14) minimum_gps_speed = tables.Int32Col(pos=15) angle_correction = tables.Float32Col(pos=16)
class DataSensor(tb.IsDescription): """Store metadata information for the SiPMs (position, gain, calibration-constant, mask). """ channel = tb.Int32Col(pos=0) # electronic channel position = tb.Float32Col(pos=1, shape=3) coeff = tb.Float64Col(pos=2) adc_to_pes = tb.Float32Col(pos=3) noise_rms = tb.Float32Col(pos=4)
class MCHitInfo(tb.IsDescription): """Stores the simulated hits as metadata using Pytables. """ hit_position = tb.Float32Col(pos=0, shape=3) hit_time = tb.Float64Col(pos=1) hit_energy = tb.Float32Col(pos=2) label = tb.StringCol(20, pos=3) particle_indx = tb.Int16Col(pos=4) hit_indx = tb.Int16Col(pos=5)
class Region(tables.IsDescription): file_key = tables.UInt32Col(pos=0) chromosome = tables.StringCol(nps.chromosome_name_length, pos=1) region_name = tables.StringCol(64, pos=2) start = tables.UInt64Col(pos=3) stop = tables.UInt64Col(pos=4) strand = tables.StringCol(1, pos=5) count = tables.UInt64Col(pos=6) normalized_count = tables.Float64Col(pos=7)
class Particle(tb.IsDescription): name = tb.StringCol(16) # 16-character String idnumber = tb.Int64Col() # Signed 64-bit integer ADCcount = tb.UInt16Col() # Unsigned short integer TDCcount = tb.UInt8Col() # unsigned byte grid_i = tb.Int32Col() # 32-bit integer grid_j = tb.Int32Col() # 32-bit integer pressure = tb.Float32Col() # float (single-precision) energy = tb.Float64Col() # double (double-precision)
class Trade(tables.IsDescription): time = tables.Int64Col() buy_order_id = tables.Int64Col() sell_order_id = tables.Int64Col() buyer = tables.Int64Col() seller = tables.Int64Col() price = tables.Float64Col() quantity = tables.Int64Col() side = tables.Int8Col()
class Particle(tables.IsDescription): ADCcount = tables.Int16Col() # signed short integer TDCcount = tables.UInt8Col() # unsigned byte grid_i = tables.Int32Col() # integer grid_j = tables.Int32Col() # integer idnumber = tables.Int64Col() # signed long long name = tables.StringCol(16, dflt="") # 16-character String pressure = tables.Float32Col(shape=2) # float (single-precision) temperature = tables.Float64Col() # double (double-precision)
class Coords_and_vel(tables.IsDescription): time = tables.Float64Col() x_pos = tables.Float64Col() vx = tables.Float64Col() y_pos = tables.Float64Col() vy = tables.Float64Col() z_pos = tables.Float64Col() vz = tables.Float64Col()
class TruthData(tables.IsDescription): E = tables.Float64Col(pos=0) x = tables.Float64Col(pos=1) y = tables.Float64Col(pos=2) z = tables.Float64Col(pos=3) px = tables.Float64Col(pos=4) py = tables.Float64Col(pos=5) pz = tables.Float64Col(pos=6)
class Timecheck(tables.IsDescription): time = tables.Float64Col() x = tables.Float64Col() vx = tables.Float64Col() y = tables.Float64Col() vy = tables.Float64Col() z = tables.Float64Col() vz = tables.Float64Col()
class info2(tables.IsDescription): y3 = tables.Float64Col(dflt=1, shape=(2, 3)) z3 = tables.UInt8Col(dflt=1) name = tables.StringCol(10) value = tables.EnumCol(colors, 'blue', base='uint32', shape=(1,)) class info3(tables.IsDescription): name = tables.StringCol(10) value = tables.Time64Col() y4 = tables.Float64Col(dflt=1, shape=(2, 3)) z4 = tables.UInt8Col(dflt=1)
class HitsTable(tb.IsDescription): event = tb. Int32Col(pos=0) time = tb.Float64Col(pos=1) npeak = tb. UInt16Col(pos=2) nsipm = tb. UInt16Col(pos=3) X = tb.Float64Col(pos=4) Y = tb.Float64Col(pos=5) Xrms = tb.Float64Col(pos=6) Yrms = tb.Float64Col(pos=7) Z = tb.Float64Col(pos=8) Q = tb.Float64Col(pos=9) E = tb.Float64Col(pos=10)
class Create(tb.IsDescription): nrows = tb.Int32Col(pos=0) irows = tb.Int32Col(pos=1) tfill = tb.Float64Col(pos=2) tidx = tb.Float64Col(pos=3) tcfill = tb.Float64Col(pos=4) tcidx = tb.Float64Col(pos=5) rowsecf = tb.Float64Col(pos=6) rowseci = tb.Float64Col(pos=7) fsize = tb.Float64Col(pos=8) isize = tb.Float64Col(pos=9) psyco = tb.BoolCol(pos=10)
class MonitoringSubarrayPointing(tables.IsDescription): """ Create the r0 Subarray pointing inside the monitoring directory """ time = tables.Float64Col() tels_with_trigger = tables.UInt64Col() array_azimuth = tables.Float32Col() array_altitude = tables.Float32Col() array_ra = tables.Float32Col() array_dec = tables.Float32Col()
class _TableBody(tables.IsDescription): name = tables.StringCol(20) additional_names = tables.StringCol(20) body_type = tables.StringCol(5) constellation = tables.StringCol(4) ra = tables.Float64Col() dec = tables.Float64Col() mag = tables.Float64Col() surface_brightness = tables.StringCol(4) size_max = tables.StringCol(8) size_min = tables.StringCol(8) positional_angle = tables.Float64Col() sci_class = tables.StringCol(11) central_star_mag = tables.Float64Col() catalog = tables.StringCol(4) ngc_descr = tables.StringCol(55) notes = tables.StringCol(86)
class ResultRecordFT(tables.IsDescription): """ Fisher's exact test """ chromosome = tables.Int32Col() position = tables.Int32Col() score = tables.Float64Col() maf = tables.Float32Col() mac = tables.Int32Col() odds_ratio = tables.Float32Col()
class HistrogramData(tables.IsDescription): energy = tables.Float64Col() number = tables.Int32Col() bins_low = tables.Float64Col(shape=bins_1.size) bins_high = tables.Float64Col(shape=bins_2.size) gamma_hist_low = tables.Float64Col(shape=bins_1.size - 1) gamma_hist_high = tables.Float64Col(shape=bins_2.size - 1) electron_hist_low = tables.Float64Col(shape=bins_1.size - 1) electron_hist_high = tables.Float64Col(shape=bins_2.size - 1)
class Trades(tables.IsDescription): time = tables.Int64Col() trader_id = tables.Int64Col() trade_id = tables.Int64Col() sequence_id = tables.Int64Col() side = tables.Int8Col() price = tables.Float64Col() quantity = tables.Int64Col() origin_id = tables.Int8Col() is_auction = tables.BoolCol() is_aggressor = tables.BoolCol()
class _PointDescriptor(tables.IsDescription): """ Descriptor for storing Point information Provides the column definition to store Point information (x, y, z). """ uid = tables.StringCol(32, pos=0) data = tables.StringCol(32, pos=1) coordinates = tables.Float64Col(pos=2, shape=(3, ))
class Info(t.IsDescription): #1 _v_pos = 1 name = t.StringCol(itemsize=2) value = t.ComplexCol(itemsize=16, pos=0) #0 y2 = t.Float64Col(dflt=1, pos=1) #1 z2 = t.UInt8Col(dflt=1) class Info2(t.IsDescription): y3 = t.Time64Col(dflt=1, shape=2) z3 = t.EnumCol({'r':4, 'g':2, 'b':1}, 'r', 'int32', shape=2) name = t.StringCol(itemsize=2) value = t.ComplexCol(itemsize=16, shape=2)
def main(): vector_size = 128 with tables.open_file('/tmp/8GB.h5', 'w') as f: table = f.create_table(f.root, 'table', { 'vector_values': tables.Float64Col(pos=0, shape=(vector_size, )), }) vector = np.zeros(vector_size) for _ in range(8300000): table.row['vector_values'] = vector table.row.append()
def __init__(self, output_filename, n_permutations): self.h5file = tables.open_file(output_filename, 'w') self.column_names = ['snp_id'] + [ 'permutation_' + str(x) for x in range(n_permutations) ] #define the permutation result object on-the-fly, depending on the number of permutations that will be performed self.permutation_result_definition = dict([ (x, tables.Float64Col()) for x in self.column_names if x.split('_')[0] == 'permutation' ]) self.permutation_result_definition['snp_id'] = tables.StringCol(100)
class PLS_temp(tb.IsDescription): """PLS table row template""" PLS_id = tb.Int16Col(dflt=0, pos=0) g1 = tb.Float64Col(dflt=0.0, pos=1) resopfrac = tb.Float64Col(dflt=0.0, pos=2) tleaf = tb.Float32Col(dflt=0.0, pos=3) twood = tb.Float32Col(dflt=0.0, pos=4) troot = tb.Float32Col(dflt=0.0, pos=5) aleaf = tb.Float32Col(dflt=0.0, pos=6) awood = tb.Float32Col(dflt=0.0, pos=7) aroot = tb.Float32Col(dflt=0.0, pos=8) c4 = tb.UInt8Col(dflt=0, pos=9) leaf_n2c = tb.Float32Col(dflt=0.0, pos=10) awood_n2c = tb.Float32Col(dflt=0.0, pos=11) froot_n2c = tb.Float32Col(dflt=0.0, pos=12) leaf_p2c = tb.Float32Col(dflt=0.0, pos=13) awood_p2c = tb.Float32Col(dflt=0.0, pos=14) froot_p2c = tb.Float32Col(dflt=0.0, pos=15) amp = tb.Float32Col(dflt=0.0, pos=16) pdia = tb.Float32Col(dflt=0.0, pos=17)
class Distribution(tb.IsDescription): """ kind of volume distribution Table structure: - date : POSIX 시간(초)을 Integer 형태로 저장 - value : 거래량 / (고가 - 저가) - price : 각 value의 column index """ row = tb.UInt64Col(pos=0) price = tb.Float64Col(pos=1) value = tb.Float16Col(pos=2)
class MetaTable(tb.IsDescription): index_start = tb.UInt32Col(pos=0) # start_index index_stop = tb.UInt32Col(pos=1) # stop_index data_length = tb.UInt32Col(pos=2) # length = tb.UInt32Col(pos=2) # https://github.com/PyTables/PyTables/issues/230 #timestamp = tb.Time64Col(pos=3) timestamp = tb.Float64Col(pos=3) error = tb.UInt32Col(pos=4)
def create_meas_group(h5f, title="measurement data", description=None, attrs=MEAS_UNITS, arrays=["data"], **kwargs): """ Create the table for saving oscilloscope measurements Parameters ---------- h5f : string or h5filehandle The file to use, if a string create or open new file title: string, optional The title description of the group data_shape: int Number of modes/polarizations description: dict or tables.IsDescription (optional) If given use to create the table arrays: list, optional name of arrays referenced in the table attrs: dict, optional attributes on the table **kwargs: keyword arguments passed to create_table/array, it is highly recommended to set expectedrows Returns ------- h5f : h5filehandle Pytables handle to the hdf file """ try: gr_meas = h5f.create_group("/", "measurements", title=title) except AttributeError: h5f = tb.open_file(h5f, "a") gr_meas = h5f.create_group("/", "measurements", title=title) gr_osc = h5f.create_group(gr_meas, "oscilloscope", title="Data from Realtime oscilloscope") if description is None: description = { "id": tb.Int64Col(), "samplingrate": tb.Float64Col(), "idx_data": tb.Int64Col() } t_meas = h5f.create_table(gr_osc, "signal", description, "sampled signal", **kwargs) setattr(t_meas.attrs, "arrays", arrays) arr = h5f.create_mdvlarray(gr_osc, "data", tb.ComplexAtom(itemsize=16), **kwargs) for k, v in attrs.items(): setattr(t_meas.attrs, k, v) return h5f
def create_recvd_data_group(h5f, title="data analysis and qampy results", description=None, oversampling_dflt=2, attrs=DSP_UNITS, arrays=["data", "symbols", "taps", "bits"], nmodes=2, **kwargs): """ Create the table for saving recovered data and parameters after DSP Parameters ---------- h5f : string or h5filehandle The file to use, if a string create or open new file title: string The title description of the group description: dict or tables.IsDescription (optional) If given use to create the table attrs: dict, optional attributes for the table arrays: list, optional name of arrays referenced in the table nmodes: int, optional number of modes/polarisations **kwargs: keyword arguments passed to create_table/array, it is highly recommended to set expectedrows Returns ------- h5f : h5filehandle Pytables handle to the hdf file """ try: gr = h5f.create_group("/", "analysis", title=title) except AttributeError: h5f = tb.open_file(h5f, "a") gr = h5f.create_group("/", "analysis", title=title) gr_dsp = h5f.create_group(gr, "qampy", title="Signal from DSP") if description is None: dsp_params = { "freq_offset": tb.Float64Col(dflt=np.nan), "freq_offset_N": tb.Int64Col(dflt=0), "phase_est": tb.StringCol(itemsize=20), "N_angles": tb.Float64Col(dflt=np.nan), "ph_est_blocklength": tb.Int64Col(), "stepsize": tb.Float64Col(shape=2), "trsyms": tb.Float64Col(shape=2), "iterations": tb.Int64Col(shape=2), "ntaps": tb.Int64Col(), "method": tb.StringCol(itemsize=20)} description = {"id":tb.Int64Col(), "idx_data": tb.Int64Col(), "idx_symbols": tb.Int64Col(), "idx_bits": tb.Int64Col(), "idx_taps": tb.Int64Col(), "evm": tb.Float64Col(dflt=np.nan, shape=nmodes), "ber":tb.Float64Col(dflt=np.nan, shape=nmodes), "ser":tb.Float64Col(dflt=np.nan, shape=nmodes), "oversampling":tb.Int64Col(dflt=oversampling_dflt)} description.update(dsp_params) t_rec = h5f.create_table(gr_dsp, "signal", description, "signal after DSP", **kwargs) setattr(t_rec.attrs, "arrays", arrays) data_arr = h5f.create_mdvlarray(gr_dsp, "data", tb.ComplexAtom(itemsize=16), "signal after DSP", **kwargs) syms_arr = h5f.create_mdvlarray(gr_dsp, "symbols", tb.ComplexAtom(itemsize=16, dflt=np.nan), "recovered symbols", **kwargs) taps_arr = h5f.create_mdvlarray(gr_dsp, "taps", tb.ComplexAtom(itemsize=16, dflt=np.nan), "qampy taps", **kwargs) bits_arr = h5f.create_mdvlarray(gr_dsp, "bits", tb.BoolAtom(dflt=False), "recovered bits", **kwargs) for k, v in attrs.items(): setattr(t_rec.attrs, k, v) return h5f