class TelTableRow(IsDescription): """Describe row format for telescope type table. Contains parameter information for each telescope type in the data. NOTE: Additional columns are added dynamically to some tables, see the github wiki page for the full table/data format descriptions. Attributes ---------- type : tables.StringCol Telescope type name (i.e. 'LST:LSTCam') optics : tables.StringCol Telescope optics type name (i.e. 'LST'). camera : tables.StringCol Telescope camera type name (i.e. 'LSTCam'). num_pixels: tables.UInt32Col Number of pixels in the telescope camera. pix_rotation: tables.Float32Col Rotation angle in deg. cam_rotation: tables.Float32Col Overall camera rotation in deg. """ type = StringCol(20) optics = StringCol(20) camera = StringCol(20) num_pixels = UInt32Col() pix_rotation = Float32Col() cam_rotation = Float32Col()
def setup(self): self.x_blob = self.file_.create_earray(self.node, "ts_x_values", Atom.from_dtype(np.dtype("int64")), (0,), filters=filters) self.y_blob = self.file_.create_earray(self.node, "ts_y_values", Atom.from_dtype(np.dtype("float64")), (0,), filters=filters) self.bp = self.file_.create_earray(self.node, "bp", Atom.from_dtype(np.dtype("int32")), (0,), filters=filters) description = {} description["unique_id"] = StringCol(itemsize=64, pos=0) description["index"] = UInt32Col(pos=1) description["blank_flags_is_none"] = BoolCol(pos=2) description["label"] = StringCol(itemsize=32, pos=3) description["start"] = UInt32Col(pos=4) description["size"] = UInt32Col(pos=5) description["bp_start"] = UInt32Col(pos=6) description["bp_size"] = UInt32Col(pos=7) self.ts_index = self.file_.create_table(self.node, "ts_index", description, filters=None) # every colums which appears in a where method call should/must be indexed ! # this is not only for performance but for correct lookup as well (I had strange bugs # else) self.ts_index.cols.unique_id.create_index() self.ts_index.cols.index.create_index()
class AgentGameOutcomeRow(IsDescription): game_number = Int32Col(pos=0) opponent = StringCol(name_size, pos=1) version = Int32Col(pos=2) moved_first = BoolCol(pos=3) outcome = StringCol(outcome_size, pos=4) when = TimeStamp()
def test_reads_meta_schema(self): temp_fs = fsopendir('temp://') filename = temp_fs.getsyspath('temp.h5') # use minimal descriptor to make the test simplier. descriptor = { 'pos': Int64Col(), 'name': StringCol(itemsize=255), 'type': StringCol(itemsize=255) } rows = [[('pos', float(i)), ('name', str(i)), ('type', str(i))] for i in range(2)] self._write_test_meta(temp_fs, 'schema', descriptor, rows) with open_file(filename, mode='r') as h5_file: ret = HDFReader._read_meta(h5_file) self.assertIn('schema', ret) self.assertEqual(len(ret['schema']), 3) # One for template, other for columns. self.assertEqual(ret['schema'][0], MPRowsFile.SCHEMA_TEMPLATE) self.assertEqual(len(ret['schema'][1]), len(MPRowsFile.SCHEMA_TEMPLATE)) self.assertEqual(len(ret['schema'][0]), len(MPRowsFile.SCHEMA_TEMPLATE)) pos_index = MPRowsFile.SCHEMA_TEMPLATE.index('pos') name_index = MPRowsFile.SCHEMA_TEMPLATE.index('name') self.assertEqual(ret['schema'][1][pos_index], 0) self.assertEqual(ret['schema'][2][pos_index], 1.0) self.assertEqual(ret['schema'][1][name_index], '0') self.assertEqual(ret['schema'][2][name_index], '1')
class Record(tables.IsDescription): var1 = StringCol(itemsize=4, dflt=b"abcd", pos=0) var2 = StringCol(itemsize=1, dflt=b"a", pos=1) var3 = BoolCol(dflt=1) var4 = Int8Col(dflt=1) var5 = UInt8Col(dflt=1) var6 = Int16Col(dflt=1) var7 = UInt16Col(dflt=1) var8 = Int32Col(dflt=1) var9 = UInt32Col(dflt=1) var10 = Int64Col(dflt=1) var11 = Float32Col(dflt=1.0) var12 = Float64Col(dflt=1.0) var13 = ComplexCol(itemsize=8, dflt=(1. + 0.j)) var14 = ComplexCol(itemsize=16, dflt=(1. + 0.j)) if hasattr(tables, 'Float16Col'): var15 = tables.Float16Col(dflt=1.0) if hasattr(tables, 'Float96Col'): var16 = tables.Float96Col(dflt=1.0) if hasattr(tables, 'Float128Col'): var17 = tables.Float128Col(dflt=1.0) if hasattr(tables, 'Complex196Col'): var18 = tables.ComplexCol(itemsize=24, dflt=(1. + 0.j)) if hasattr(tables, 'Complex256Col'): var19 = tables.ComplexCol(itemsize=32, dflt=(1. + 0.j))
def _get_rows_descriptor(columns): """ Converts columns specifications from ambry_sources format to pytables descriptor. Args: columns (list of dict) Returns: dict: valid pytables descriptor. """ TYPE_MAP = { 'int': lambda pos: Int32Col(pos=pos), 'long': lambda pos: Int64Col(pos=pos), 'str': lambda pos: StringCol(itemsize=255, pos=pos), 'bytes': lambda pos: StringCol(itemsize=255, pos=pos), 'float': lambda pos: Float64Col(pos=pos), 'unknown': lambda pos: StringCol(itemsize=255, pos=pos), } descriptor = {} for column in columns: pytables_type = TYPE_MAP.get(column['type']) if not pytables_type: raise Exception( 'Failed to convert `{}` ambry_sources type to pytables type.'. format(column['type'])) descriptor[column['name']] = pytables_type(column['pos']) return descriptor
class SessionMetaData(tables.IsDescription): session_id = UInt32Col(pos=1) experiment_id = UInt32Col(pos=2) code = StringCol(24, pos=3) name = StringCol(48, pos=4) comments = StringCol(256, pos=5) user_variables = StringCol(2048, pos=6) # will hold json encoded version of user variable dict for session
class ExperimentMetaData(tables.IsDescription): experiment_id = UInt32Col(pos=1) code = StringCol(24, pos=2) title = StringCol(48, pos=3) description = StringCol(256, pos=4) version = StringCol(6, pos=5) total_sessions_to_run = UInt16Col(pos=9)
class SpeciesTable(tables.IsDescription): """ 2013.3.8 """ id = UInt64Col(pos=0) name = StringCol(512, pos=1) scientific_name = StringCol(512, pos=2) ploidy = UInt16Col(pos=3)
class IndividualTable(tables.IsDescription): id = UInt64Col(pos=0) family_id = StringCol(512, pos=1) #64 byte-long name = StringCol(512, pos=2) # name should look like 'species_name.population_name.individual_name' to ensure uniqueness father_name = StringCol(512, pos=3) mother_name = StringCol(512, pos=4) sex = UInt64Col(pos=5) #0 is unknown, 1=male, 2=female phenotype = Float64Col(pos=6) population_id = UInt64Col(pos=7)
def _save_source(self): descriptor = { 'fetch_time': Float64Col(), 'encoding': StringCol(itemsize=255), 'url': StringCol(itemsize=1024), 'file_type': StringCol(itemsize=50), 'inner_file': StringCol(itemsize=255), 'url_type': StringCol(itemsize=255), } self._save_meta_child('source', descriptor)
class GameSummaryAgent(IsDescription): name = StringCol(name_size, pos=0) version = Int32Col(pos=1) game_number = Int32Col(pos=2) rating = Float64Col(pos=3, dflt=0.0) outcome = StringCol(outcome_size, pos=4) total_time = Float64Col(pos=5) time_med = Float64Col(pos=6) time_max = Float64Col(pos=7) state_size_med = Int32Col(pos=8) state_size_max = Int32Col(pos=9)
def _save_row_spec(self): descriptor = { 'end_row': Int32Col(), 'header_rows': StringCol(itemsize=255), # comma separated ints or empty string. 'start_row': Int32Col(), 'comment_rows': StringCol(itemsize=255), # comma separated ints or empty string. 'data_pattern': StringCol(itemsize=255) } self._save_meta_child('row_spec', descriptor)
class _FeatureDescription(IsDescription): seqid = StringCol(256) start = Int32Col() end = Int32Col() Name = StringCol(256) biotype = StringCol(256)
class ChromosomeTable(tables.IsDescription): """ 2013.3.8 """ id = UInt64Col(pos=0) name = StringCol(512, pos=1) #should be unique, but not 100% enforced #name should look like species_name.chromosome_name, if species_name is available length = UInt64Col(pos=2) sex_chromosome = StringCol(4, pos=3) #A=autosome, X=chr X, Y=chr Y species_id = UInt64Col(pos=4) path = StringCol(4512, pos=5) #path to the file containing the chromosome sequences, should be unique,
class PolymorphismTable(tables.IsDescription): """ 2013.3.6 table that records the polymorphism alleles of individuals at relevant loci """ id = UInt64Col(pos=0) name = StringCol(512, pos=1) #name should be "individualName.locusName.chromosome_copy" to ensure uniqueness individual_id = UInt64Col(pos=2) locus_id = UInt64Col(pos=3) chromosome_copy = UInt64Col(pos=4, dflt=0) #starting from 0 allele_sequence = StringCol(512, pos=5) allele_sequence_length = UInt64Col(pos=6) allele_type = StringCol(512, pos=7)
class AssociationLocusTable(tables.IsDescription): """ 2013.1.28 bugfix, was pos=3 for no_of_peaks (same as stop), now change it to pos=4, and increment others accordingly """ id = UInt64Col(pos=0) chromosome = StringCol(64, pos=1) #64 byte-long start = UInt64Col(pos=2) stop = UInt64Col(pos=3) no_of_peaks = UInt64Col(pos=4) connectivity = Float64Col(pos=5) no_of_results = UInt64Col(pos=6) phenotype_id_ls_in_str = StringCol(1000, pos=7)
class CountAssociationLocusTable(tables.IsDescription): """ 2012.12.24 new PyTables-based table definition """ id = UInt64Col(pos=0) min_score = Float64Col(pos=1) min_overlap_ratio = Float64Col(pos=2) total_no_of_results = UInt64Col(pos=3) no_of_association_loci = UInt64Col(pos=4) call_method_id_ls = StringCol(1000, pos=5) cnv_method_id_ls = StringCol(1000, pos=6) phenotype_method_id_ls = StringCol(1000, pos=7) analysis_method_id_ls = StringCol(1000, pos=8)
class Preopen(IsDescription): symbol = StringCol(50) xDt = StringCol(50) caAct = StringCol(50) iep = Float32Col() chn = Float16Col() perChn = Float16Col() pCls = Float32Col() trdQnty = Int32Col() iVal = Float16Col() sumVal = Float32Col() sumQnty = Int32Col() finQnty = Int32Col() sumfinQnty = Int32Col()
class Equity(IsDescription): SYMBOL = StringCol(50) SERIES = StringCol(2) OPEN = Float32Col() HIGH = Float32Col() LOW = Float32Col() CLOSE = Float32Col() LAST = Float32Col() PREVCLOSE = Float32Col() TOTTRDQTY = Int32Col() TOTTRDVAL = Float64Col() TIMESTAMP = StringCol(12) TOTALTRADES = Int32Col() ISIN = StringCol(12)
def saveField(h5, resultGroup, result): def dump(inputList): def conversion(arg): if type(arg) == type(u' '): return arg.encode('utf-8') else: return arg.__repr__() if type(inputList) == type([]): return map(conversion, inputList) else: return map(dump, inputList) if result.data.dtype.char in ['U', 'O']: unicodeData = scipy.array(dump(result.data.tolist())) h5.createArray(resultGroup, "data", unicodeData, result.longname.encode("utf-8")) else: h5.createArray(resultGroup, "data", result.data, result.longname.encode("utf-8")) for key, value in result.attributes.iteritems(): h5.setNodeAttr(resultGroup.data, key, value) h5.setNodeAttr(resultGroup, "longname", result.longname.encode("utf-8")) h5.setNodeAttr(resultGroup, "shortname", result.shortname.encode("utf-8")) h5.setNodeAttr(resultGroup, "creator", result.creator.encode("utf-8")) h5.setNodeAttr(resultGroup, "machine", result.machine.encode("utf-8")) if result.error != None: h5.createArray(resultGroup, "error", result.error, (u"Error of " + result.longname).encode("utf-8")) if result.mask != None: h5.createArray(resultGroup, "mask", result.mask, (u"Mask of " + result.longname).encode("utf-8")) h5.setNodeAttr(resultGroup, "unit", repr(result.unit).encode("utf-8")) if result.dimensions != DataContainer.INDEX: idLen = max([len(dim.id.encode("utf-8")) for dim in result.dimensions]) dimTable = h5.createTable(resultGroup, "dimensions", { "hash": StringCol(32), "id": StringCol(idLen) }, (u"Dimensions of " + result.longname).encode("utf-8"), expectedrows=len(result.dimensions)) for dim in result.dimensions: d = dimTable.row d["hash"] = dim.hash.encode("utf-8") d["id"] = dim.id.encode("utf-8") d.append() saveResult(dim, h5) dimTable.flush()
class Derivative(IsDescription): INSTRUMENT = StringCol(6) SYMBOL = StringCol(50) EXPIRY_DT = StringCol(11) STRIKE_PR = Int32Col() OPTION_TYP = StringCol(2) OPEN = Float32Col() HIGH = Float32Col() LOW = Float32Col() CLOSE = Float32Col() SETTLE_PR = Float32Col() CONTRACTS = Int32Col() VAL_INLAKH = Float64Col() OPEN_INT = Int32Col() CHG_IN_OI = Int32Col() TIMESTAMP = StringCol(11)
class OXTS(IsDescription): """HFS5 OXTS table model""" timestamp = StringCol(32) lat = Float64Col() lon = Float64Col() alt = Float64Col() roll = Float64Col() pitch = Float64Col() yaw = Float64Col() vn = Float64Col() ve = Float64Col() vf = Float64Col() vl = Float64Col() vu = Float64Col() ax = Float64Col() ay = Float64Col() az = Float64Col() af = Float64Col() al = Float64Col() au = Float64Col() wx = Float64Col() wy = Float64Col() wz = Float64Col() wf = Float64Col() wl = Float64Col() wu = Float64Col() pos_accuracy = Float64Col() vel_accuracy = Float64Col() navstat = UInt8Col() numsats = UInt8Col() posmode = UInt8Col() velmode = UInt8Col() orimode = UInt8Col()
class Array(IsDescription): """Row descriptor class for Pytables array data table. Contains parameter information for each selected telescope in the data. Attributes ---------- tel_id : UInt8Col UInt8 placeholder type for the telescope id (in the array) tel_x : Float32Col Float32 placeholder type for the telescope position x coordinate relative to the center of the array. tel_y : Float32Col Float32 placeholder type for the telescope position y coordinate relative to the center of the array. tel_z : Float32Col Float32 placeholder type for the telescope position z coordinate (height) relative to the CORSIKA observatory altitude. tel_type : StringCol String placeholder type for the telescope type name (i.e. 'LST') run_array_direction: Float32Col(2) Float32 tuple placeholder type for the array pointing direction for a given run (az,alt) """ tel_id = UInt8Col() tel_x = Float32Col() tel_y = Float32Col() tel_z = Float32Col() tel_type = StringCol(8) run_array_direction = Float32Col(2)
def __init__(self, p, collength=16, verbose=False): """ Instantiates a MAP object corresponding to MAP files. The data_format is how the file is set up for use with reading and writing HDF5 tables. The dictionary key represents the name of the column, the first part of the value tuple represents where it is located when the line is split and the second part represents the datatype to use within the table. :param p: The full file path """ self.data_format = { "chromosome": (0, StringCol(collength)), "identifier": (1, StringCol(collength)), "distance": (2, StringCol(collength)), "position": (3, StringCol(collength)) } DataType.__init__(self, p, "map", verbose)
class ArrayTableRow(IsDescription): """Describe row format for telescope array table. Contains parameter information for each telescope in the array. NOTE: Additional columns are added dynamically to some tables, see the github wiki page for the full table/data format descriptions. Attributes ---------- id : tables.UInt8Col Telescope id (unique). type : tables.StringCol Telescope type name (i.e. 'LST:LSTCam'). x : tables.Float32Col Telescope position x coordinate relative to the center of the array. y : tables.Float32Col Telescope position y coordinate relative to the center of the array. z : tables.Float32Col Telescope position z coordinate (height) relative to the CORSIKA observatory altitude. """ id = UInt16Col() type = StringCol(20) x = Float32Col() y = Float32Col() z = Float32Col()
class PopulationTable(tables.IsDescription): """ 2013.3.8 """ id = UInt64Col(pos=0) name = StringCol(512, pos=1) #name should like 'species_name.population_name' size = UInt64Col(pos=2) species_id = UInt64Col(pos=3)
def test_converts_comment_rows_and_header_rows_json_to_list(self): temp_fs = fsopendir('temp://') # save meta.row_spec to the file. descriptor = { # this is not valid descriptor, but I do not need it to be valid here. 'header_rows': StringCol(itemsize=255), 'comment_rows': StringCol(itemsize=255), } self._write_test_meta(temp_fs, 'row_spec', descriptor, [[('comment_rows', json.dumps([0, 1])), ('header_rows', json.dumps([2, 3]))]]) # now read it from file. with open_file(temp_fs.getsyspath('temp.h5'), 'r') as h5: ret = HDFReader._read_meta_child(h5, 'row_spec') self.assertEqual(ret[0]['comment_rows'], [0, 1]) self.assertEqual(ret[0]['header_rows'], [2, 3])
def test_converts_hist_and_uvalues_json_to_list(self): temp_fs = fsopendir('temp://') # save meta.schema minimal table to the file. descriptor = { # this is not valid descriptor, but I do not need it to be valid here. 'hist': StringCol(itemsize=255), 'uvalues': StringCol(itemsize=255), } self._write_test_meta(temp_fs, 'schema', descriptor, [[('hist', json.dumps([0, 1])), ('uvalues', json.dumps(['a', 'b']))]]) # now read it from file. with open_file(temp_fs.getsyspath('temp.h5'), 'r') as h5: ret = HDFReader._read_meta_child(h5, 'schema') self.assertEqual(ret[0]['hist'], [0, 1]) self.assertEqual(ret[0]['uvalues'], ['a', 'b'])
class CameraScanTableDescription(IsDescription): """ """ setpoint = Float32Col() frame_path = StringCol(140) ravg = Float32Col() gavg = Float32Col() bavg = Float32Col()