class AgentGameOutcomeRow(IsDescription): game_number = Int32Col(pos=0) opponent = StringCol(name_size, pos=1) version = Int32Col(pos=2) moved_first = BoolCol(pos=3) outcome = StringCol(outcome_size, pos=4) when = TimeStamp()
def _create_h5(filename): # save meta.about to the file. with open_file(filename, 'w') as h5: descriptor1 = { 'field1': Float64Col(), 'field2': Float64Col(), 'field3': Float64Col() } descriptor2 = { 'version': Int32Col(), 'n_rows': Int32Col(), 'n_cols': Int32Col() } h5.create_group('/partition', 'meta', createparents=True) rows_table = h5.create_table('/partition', 'rows', descriptor1) file_header_table = h5.create_table('/partition', 'file_header', descriptor2) for i in range(5): rows_table.row['field1'] = float(i) rows_table.row['field2'] = float(i) rows_table.row['field2'] = float(i) rows_table.row.append() rows_table.flush() file_header_table.row['version'] = 1 file_header_table.row['n_rows'] = 1 file_header_table.row['n_cols'] = 1 file_header_table.row.append() file_header_table.flush()
class AgentVersionRow(IsDescription): version = Int32Col(pos=0, dflt=0) won = Int32Col(pos=1, dflt=0) lost = Int32Col(pos=2, dflt=0) drawn = Int32Col(pos=3, dflt=0) failed = Int32Col(pos=4, dflt=0) rating = Float64Col(pos=5, dflt=0.0) uploaded = TimeStamp()
class Particle(IsDescription): name = StringCol(16) # 16-character String idnumber = Int64Col() # Signed 64-bit integer ADCcount = UInt16Col() # Unsigned short integer TDCcount = UInt8Col() # unsigned byte grid_i = Int32Col() # 32-bit integer grid_j = Int32Col() # 32-bit integer pressure = Float32Col() # float (single-precision) energy = Float64Col() # double (double-precision)
class GameSummaryAgent(IsDescription): name = StringCol(name_size, pos=0) version = Int32Col(pos=1) game_number = Int32Col(pos=2) rating = Float64Col(pos=3, dflt=0.0) outcome = StringCol(outcome_size, pos=4) total_time = Float64Col(pos=5) time_med = Float64Col(pos=6) time_max = Float64Col(pos=7) state_size_med = Int32Col(pos=8) state_size_max = Int32Col(pos=9)
def _save_row_spec(self): descriptor = { 'end_row': Int32Col(), 'header_rows': StringCol(itemsize=255), # comma separated ints or empty string. 'start_row': Int32Col(), 'comment_rows': StringCol(itemsize=255), # comma separated ints or empty string. 'data_pattern': StringCol(itemsize=255) } self._save_meta_child('row_spec', descriptor)
class _FeatureDescription(IsDescription): seqid = StringCol(256) start = Int32Col() end = Int32Col() Name = StringCol(256) biotype = StringCol(256)
class McmcParameters(IsDescription): """ Class that holds the data model for the HDF5 table with the MCMC parameters. """ iterations = Int32Col() burnIn = Int32Col() thin = Int32Col() minMeanAbsoluteMagnitude = Float64Col() maxMeanAbsoluteMagnitude = Float64Col() priorTau = StringCol(40) shapeTau = Float64Col() scaleTau = Float64Col()
class Post(IsDescription): id = Int32Col() post_type_id = UInt8Col() parent_id = Int32Col() accepted_answer_id = Int32Col() creation_date = Time32Col() score = UInt32Col() viewcount = UInt32Col() owner_user_id = Int32Col() answer_count = UInt16Col() favorite_count = UInt16Col() tags = StringCol(itemsize=25, shape=(5, ))
class Equity(IsDescription): SYMBOL = StringCol(50) SERIES = StringCol(2) OPEN = Float32Col() HIGH = Float32Col() LOW = Float32Col() CLOSE = Float32Col() LAST = Float32Col() PREVCLOSE = Float32Col() TOTTRDQTY = Int32Col() TOTTRDVAL = Float64Col() TIMESTAMP = StringCol(12) TOTALTRADES = Int32Col() ISIN = StringCol(12)
class Preopen(IsDescription): symbol = StringCol(50) xDt = StringCol(50) caAct = StringCol(50) iep = Float32Col() chn = Float16Col() perChn = Float16Col() pCls = Float32Col() trdQnty = Int32Col() iVal = Float16Col() sumVal = Float32Col() sumQnty = Int32Col() finQnty = Int32Col() sumfinQnty = Int32Col()
class Record(tables.IsDescription): var1 = StringCol(itemsize=4, dflt=b"abcd", pos=0) var2 = StringCol(itemsize=1, dflt=b"a", pos=1) var3 = BoolCol(dflt=1) var4 = Int8Col(dflt=1) var5 = UInt8Col(dflt=1) var6 = Int16Col(dflt=1) var7 = UInt16Col(dflt=1) var8 = Int32Col(dflt=1) var9 = UInt32Col(dflt=1) var10 = Int64Col(dflt=1) var11 = Float32Col(dflt=1.0) var12 = Float64Col(dflt=1.0) var13 = ComplexCol(itemsize=8, dflt=(1. + 0.j)) var14 = ComplexCol(itemsize=16, dflt=(1. + 0.j)) if hasattr(tables, 'Float16Col'): var15 = tables.Float16Col(dflt=1.0) if hasattr(tables, 'Float96Col'): var16 = tables.Float96Col(dflt=1.0) if hasattr(tables, 'Float128Col'): var17 = tables.Float128Col(dflt=1.0) if hasattr(tables, 'Complex196Col'): var18 = tables.ComplexCol(itemsize=24, dflt=(1. + 0.j)) if hasattr(tables, 'Complex256Col'): var19 = tables.ComplexCol(itemsize=32, dflt=(1. + 0.j))
class _VariantDescription(IsDescription): CHROM = StringCol(256) POS = Int32Col() ID = StringCol(256) REF = StringCol(256) ALT = StringCol(256) QUAL = Float32Col() CAF = StringCol(256) CLNDISDB = StringCol(256) CLNDN = StringCol(256) CLNSIG = StringCol(256) CLNREVSTAT = StringCol(256) CLNVI = StringCol(256) effect = StringCol(256) impact = StringCol(256) gene_name = StringCol(256) GT = StringCol(256)
class Test_GrainData(IsDescription): """ Description class specifying structured storage for tests """ idnumber = Int32Col() # Signed 64-bit integer volume = Float32Col() # float center = Float32Col(shape=(3, )) # float
def _get_rows_descriptor(columns): """ Converts columns specifications from ambry_sources format to pytables descriptor. Args: columns (list of dict) Returns: dict: valid pytables descriptor. """ TYPE_MAP = { 'int': lambda pos: Int32Col(pos=pos), 'long': lambda pos: Int64Col(pos=pos), 'str': lambda pos: StringCol(itemsize=255, pos=pos), 'bytes': lambda pos: StringCol(itemsize=255, pos=pos), 'float': lambda pos: Float64Col(pos=pos), 'unknown': lambda pos: StringCol(itemsize=255, pos=pos), } descriptor = {} for column in columns: pytables_type = TYPE_MAP.get(column['type']) if not pytables_type: raise Exception( 'Failed to convert `{}` ambry_sources type to pytables type.'. format(column['type'])) descriptor[column['name']] = pytables_type(column['pos']) return descriptor
class Particle(IsDescription): particle_id = Int32Col() mass = Float64Col() x_pos = Float64Col() y_pos = Float64Col() x_vel = Float64Col() y_vel = Float64Col()
class Derivative(IsDescription): INSTRUMENT = StringCol(6) SYMBOL = StringCol(50) EXPIRY_DT = StringCol(11) STRIKE_PR = Int32Col() OPTION_TYP = StringCol(2) OPEN = Float32Col() HIGH = Float32Col() LOW = Float32Col() CLOSE = Float32Col() SETTLE_PR = Float32Col() CONTRACTS = Int32Col() VAL_INLAKH = Float64Col() OPEN_INT = Int32Col() CHG_IN_OI = Int32Col() TIMESTAMP = StringCol(11)
class User(IsDescription): id = Int32Col() name = StringCol(16) age = UInt8Col() reputation = UInt16Col() creation_date = Time32Col() views = UInt16Col() upvotes = UInt16Col() downvotes = UInt16Col()
class SurveyParameters(IsDescription): """ Class that holds the data model for the simulated parallax survey parameters. Intended for use with the HDF5 files through the pytables package. """ kind = StringCol(itemsize=40) numberOfStars = Int32Col() minParallax = Float64Col() maxParallax = Float64Col() meanAbsoluteMagnitude = Float64Col() varianceAbsoluteMagnitude = Float64Col() parallaxErrorNormalizationMagnitude = Float64Col() parallaxErrorSlope = Float64Col() parallaxErrorCalibrationFloor = Float64Col() magnitudeErrorNormalizationMagnitude = Float64Col() magnitudeErrorSlope = Float64Col() magnitudeErrorCalibrationFloor = Float64Col() apparentMagnitudeLimit = Float64Col() numberOfStarsInSurvey = Int32Col()
def test_saves_header_rows_and_comment_rows_as_json_string(self): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) writer._validate_groups() writer.meta['row_spec']['comment_rows'] = [0, 1] writer.meta['row_spec']['header_rows'] = [2, 3] descriptor = { 'end_row': Int32Col(), 'header_rows': StringCol(itemsize=255), 'start_row': Int32Col(), 'comment_rows': StringCol(itemsize=255), 'data_pattern': StringCol(itemsize=255) } writer._save_meta_child('row_spec', descriptor) self.assertEqual( [(x['comment_rows'], x['header_rows']) for x in writer._h5_file.root.partition.meta.row_spec.iterrows()], [(b('[0, 1]'), b('[2, 3]'))])
def write_file_header(self): """ Write the version, number of rows and number of cols to the h5 file. """ if 'file_header' in self._h5_file.root.partition: self._h5_file.remove_node('/partition', 'file_header') descriptor = { 'version': Int32Col(), 'n_rows': Int32Col(), 'n_cols': Int32Col() } table = self._h5_file.create_table('/partition', 'file_header', descriptor, 'Header of the file.') table.row['version'] = HDFPartition.VERSION table.row['n_rows'] = self.n_rows table.row['n_cols'] = self.n_cols table.row.append() table.flush()
class SSelectorOnMeshElement(IsDescription): shortName = StringCol(ELEMENT_LENGTH, pos=0) imin = Int32Col(pos=1) jmin = Int32Col(pos=2) kmin = Int32Col(pos=3) imax = Int32Col(pos=4) jmax = Int32Col(pos=5) kmax = Int32Col(pos=6) v1 = Float32Col() v2 = Float32Col() v3 = Float32Col()
def setup_spec_table(self): if not hasattr(self.node, "ms1_spec_table"): for level in (1, 2): description = {} description["pm_index"] = UInt32Col(pos=0) description["rt"] = Float32Col(pos=1) description["scan_number"] = Int32Col(pos=2) description["start"] = UInt64Col(pos=3) description["end"] = UInt64Col(pos=4) t = self.file_.create_table(self.node, 'ms%d_spec_table' % level, description, filters=filters) # every colums which appears in a where method call should/must be indexed ! # this is not only for performance but for correct lookup as well (I had strange bugs # else) t.cols.pm_index.create_index() t.cols.rt.create_index()
class TestTDescr(tables.IsDescription): """A description that has several nested columns.""" x = Int32Col(dflt=0, shape=2, pos=0) # 0 y = FloatCol(dflt=1, shape=(2, 2)) z = UInt8Col(dflt=1) z3 = EnumCol({'r': 4, 'g': 2, 'b': 1}, 'r', 'int32', shape=2) color = StringCol(itemsize=4, dflt=b"ab", pos=2) info = Info() class Info(tables.IsDescription): # 1 _v_pos = 1 name = StringCol(itemsize=2) value = ComplexCol(itemsize=16, pos=0) # 0 y2 = FloatCol(pos=1) # 1 z2 = UInt8Col() class Info2(tables.IsDescription): y3 = Time64Col(shape=2) name = StringCol(itemsize=2) value = ComplexCol(itemsize=16, shape=2)
class _VariantDescription(IsDescription): """ Describe VariantHDF5 table columns. """ # TODO: Optimize CHROM = StringCol(8) POS = Int32Col() ID = StringCol(16) REF = StringCol(256) ALT = StringCol(256) QUAL = Float32Col() # INFO CAF = StringCol(16) CLNSIG = StringCol(8) CLNDBN = StringCol(256) # INFO ANN effect = StringCol(32) impact = StringCol(32) gene_name = StringCol(32) # FORMAT & sample GT = StringCol(8)
class description(IsDescription): time = Int32Col() st = Int8Col(shape=(N_CH, )) wave = Float32Col(shape=(S_TOTAL, N_CH)) fet = Float32Col(shape=(N_CH, FPC)) clu = Int32Col()
class Video(IsDescription): label = Int32Col() frames = Float32Col(shape=(29, 112, 112)) yaw = Float32Col() file = StringCol(32) word = StringCol(32)
class ParametersTableRow(IsDescription): """Describe row format for parameter table. Contains parameters values for each image of each event of each telescope in the array. Parameters are calculated after image cleaning (i.e. with for example tailcut_clean method) There are Hillas, Leakage, Concentration, Timing and Morphology parameters. Attributes ---------- event_index : tables.Int32Col Event id of file (from -1 to N ) leakage_* : tables.Float32Col see at https://cta-observatory.github.io/ctapipe/api/ctapipe.containers.LeakageContainer.html?highlight=leakagecontainer#ctapipe.containers.LeakageContainer hillas_* : tables.Float32Col see at https://cta-observatory.github.io/ctapipe/api/ctapipe.containers.HillasParametersContainer.html#ctapipe.containers.HillasParametersContainer concentration_* : see at https://cta-observatory.github.io/ctapipe/api/ctapipe.containers.ConcentrationContainer.html#ctapipe.containers.ConcentrationContainer timing_* : see at https://cta-observatory.github.io/ctapipe/api/ctapipe.containers.TimingParametersContainer.html#ctapipe.containers.TimingParametersContainer morphology_* : see at https://cta-observatory.github.io/ctapipe/api/ctapipe.containers.MorphologyContainer.html#ctapipe.containers.MorphologyContainer log_hillas_intensity : tables.Float32Col logaritmic hillas intensity """ event_index = Int32Col() leakage_intensity_width_1 = Float32Col() leakage_intensity_width_2 = Float32Col() leakage_pixels_width_1 = Float32Col() leakage_pixels_width_2 = Float32Col() hillas_intensity = Float32Col() log_hillas_intensity = Float32Col() hillas_x = Float32Col() hillas_y = Float32Col() hillas_r = Float32Col() hillas_phi = Float32Col() hillas_width = Float32Col() hillas_length = Float32Col() hillas_psi = Float32Col() hillas_skewness = Float32Col() hillas_kurtosis = Float32Col() concentration_cog = Float32Col() concentration_core = Float32Col() concentration_pixel = Float32Col() timing_slope = Float32Col() # time gradient timing_slope_err = Float32Col() timing_intercept = Float32Col() timing_intercept_err = Float32Col() timing_deviation = Float32Col() morphology_num_pixels = Int32Col() morphology_num_islands = Int32Col() morphology_num_small_islands = Int32Col() morphology_num_medium_islands = Int32Col() morphology_num_large_islands = Int32Col()
class Event(IsDescription): id = Int32Col() timestamp = Time32Col() event_type = UInt8Col() event_id = Int32Col()
class Vote(IsDescription): id = Int32Col() post_id = Int32Col() vote_type_id = UInt8Col() creation_date = Time32Col()