class TelTableRow(IsDescription):
"""Describe row format for telescope type table.
Contains parameter information for each telescope type in the data. NOTE:
Additional columns are added dynamically to some tables,
see the github wiki page for the full table/data format descriptions.
Attributes
----------
type : tables.StringCol
Telescope type name (i.e. 'LST:LSTCam')
optics : tables.StringCol
Telescope optics type name (i.e. 'LST').
camera : tables.StringCol
Telescope camera type name (i.e. 'LSTCam').
num_pixels: tables.UInt32Col
Number of pixels in the telescope camera.
pix_rotation: tables.Float32Col
Rotation angle in deg.
cam_rotation: tables.Float32Col
Overall camera rotation in deg.
"""
type = StringCol(20)
optics = StringCol(20)
camera = StringCol(20)
num_pixels = UInt32Col()
pix_rotation = Float32Col()
cam_rotation = Float32Col()
def setup(self):
self.x_blob = self.file_.create_earray(self.node, "ts_x_values",
Atom.from_dtype(np.dtype("int64")), (0,),
filters=filters)
self.y_blob = self.file_.create_earray(self.node, "ts_y_values",
Atom.from_dtype(np.dtype("float64")), (0,),
filters=filters)
self.bp = self.file_.create_earray(self.node, "bp",
Atom.from_dtype(np.dtype("int32")), (0,),
filters=filters)
description = {}
description["unique_id"] = StringCol(itemsize=64, pos=0)
description["index"] = UInt32Col(pos=1)
description["blank_flags_is_none"] = BoolCol(pos=2)
description["label"] = StringCol(itemsize=32, pos=3)
description["start"] = UInt32Col(pos=4)
description["size"] = UInt32Col(pos=5)
description["bp_start"] = UInt32Col(pos=6)
description["bp_size"] = UInt32Col(pos=7)
self.ts_index = self.file_.create_table(self.node, "ts_index", description,
filters=None)
# every colums which appears in a where method call should/must be indexed !
# this is not only for performance but for correct lookup as well (I had strange bugs
# else)
self.ts_index.cols.unique_id.create_index()
self.ts_index.cols.index.create_index()
class AgentGameOutcomeRow(IsDescription):
game_number = Int32Col(pos=0)
opponent = StringCol(name_size, pos=1)
version = Int32Col(pos=2)
moved_first = BoolCol(pos=3)
outcome = StringCol(outcome_size, pos=4)
when = TimeStamp()
def test_reads_meta_schema(self):
temp_fs = fsopendir('temp://')
filename = temp_fs.getsyspath('temp.h5')
# use minimal descriptor to make the test simplier.
descriptor = {
'pos': Int64Col(),
'name': StringCol(itemsize=255),
'type': StringCol(itemsize=255)
}
rows = [[('pos', float(i)), ('name', str(i)), ('type', str(i))]
for i in range(2)]
self._write_test_meta(temp_fs, 'schema', descriptor, rows)
with open_file(filename, mode='r') as h5_file:
ret = HDFReader._read_meta(h5_file)
self.assertIn('schema', ret)
self.assertEqual(len(ret['schema']),
3) # One for template, other for columns.
self.assertEqual(ret['schema'][0], MPRowsFile.SCHEMA_TEMPLATE)
self.assertEqual(len(ret['schema'][1]),
len(MPRowsFile.SCHEMA_TEMPLATE))
self.assertEqual(len(ret['schema'][0]),
len(MPRowsFile.SCHEMA_TEMPLATE))
pos_index = MPRowsFile.SCHEMA_TEMPLATE.index('pos')
name_index = MPRowsFile.SCHEMA_TEMPLATE.index('name')
self.assertEqual(ret['schema'][1][pos_index], 0)
self.assertEqual(ret['schema'][2][pos_index], 1.0)
self.assertEqual(ret['schema'][1][name_index], '0')
self.assertEqual(ret['schema'][2][name_index], '1')
class Record(tables.IsDescription):
var1 = StringCol(itemsize=4, dflt=b"abcd", pos=0)
var2 = StringCol(itemsize=1, dflt=b"a", pos=1)
var3 = BoolCol(dflt=1)
var4 = Int8Col(dflt=1)
var5 = UInt8Col(dflt=1)
var6 = Int16Col(dflt=1)
var7 = UInt16Col(dflt=1)
var8 = Int32Col(dflt=1)
var9 = UInt32Col(dflt=1)
var10 = Int64Col(dflt=1)
var11 = Float32Col(dflt=1.0)
var12 = Float64Col(dflt=1.0)
var13 = ComplexCol(itemsize=8, dflt=(1. + 0.j))
var14 = ComplexCol(itemsize=16, dflt=(1. + 0.j))
if hasattr(tables, 'Float16Col'):
var15 = tables.Float16Col(dflt=1.0)
if hasattr(tables, 'Float96Col'):
var16 = tables.Float96Col(dflt=1.0)
if hasattr(tables, 'Float128Col'):
var17 = tables.Float128Col(dflt=1.0)
if hasattr(tables, 'Complex196Col'):
var18 = tables.ComplexCol(itemsize=24, dflt=(1. + 0.j))
if hasattr(tables, 'Complex256Col'):
var19 = tables.ComplexCol(itemsize=32, dflt=(1. + 0.j))
def _get_rows_descriptor(columns):
""" Converts columns specifications from ambry_sources format to pytables descriptor.
Args:
columns (list of dict)
Returns:
dict: valid pytables descriptor.
"""
TYPE_MAP = {
'int': lambda pos: Int32Col(pos=pos),
'long': lambda pos: Int64Col(pos=pos),
'str': lambda pos: StringCol(itemsize=255, pos=pos),
'bytes': lambda pos: StringCol(itemsize=255, pos=pos),
'float': lambda pos: Float64Col(pos=pos),
'unknown': lambda pos: StringCol(itemsize=255, pos=pos),
}
descriptor = {}
for column in columns:
pytables_type = TYPE_MAP.get(column['type'])
if not pytables_type:
raise Exception(
'Failed to convert `{}` ambry_sources type to pytables type.'.
format(column['type']))
descriptor[column['name']] = pytables_type(column['pos'])
return descriptor
class SessionMetaData(tables.IsDescription):
session_id = UInt32Col(pos=1)
experiment_id = UInt32Col(pos=2)
code = StringCol(24, pos=3)
name = StringCol(48, pos=4)
comments = StringCol(256, pos=5)
user_variables = StringCol(2048, pos=6) # will hold json encoded version of user variable dict for session
class ExperimentMetaData(tables.IsDescription):
experiment_id = UInt32Col(pos=1)
code = StringCol(24, pos=2)
title = StringCol(48, pos=3)
description = StringCol(256, pos=4)
version = StringCol(6, pos=5)
total_sessions_to_run = UInt16Col(pos=9)
class SpeciesTable(tables.IsDescription): """ 2013.3.8 """ id = UInt64Col(pos=0) name = StringCol(512, pos=1) scientific_name = StringCol(512, pos=2) ploidy = UInt16Col(pos=3)
class IndividualTable(tables.IsDescription): id = UInt64Col(pos=0) family_id = StringCol(512, pos=1) #64 byte-long name = StringCol(512, pos=2) # name should look like 'species_name.population_name.individual_name' to ensure uniqueness father_name = StringCol(512, pos=3) mother_name = StringCol(512, pos=4) sex = UInt64Col(pos=5) #0 is unknown, 1=male, 2=female phenotype = Float64Col(pos=6) population_id = UInt64Col(pos=7)
def _save_source(self):
descriptor = {
'fetch_time': Float64Col(),
'encoding': StringCol(itemsize=255),
'url': StringCol(itemsize=1024),
'file_type': StringCol(itemsize=50),
'inner_file': StringCol(itemsize=255),
'url_type': StringCol(itemsize=255),
}
self._save_meta_child('source', descriptor)
class GameSummaryAgent(IsDescription):
name = StringCol(name_size, pos=0)
version = Int32Col(pos=1)
game_number = Int32Col(pos=2)
rating = Float64Col(pos=3, dflt=0.0)
outcome = StringCol(outcome_size, pos=4)
total_time = Float64Col(pos=5)
time_med = Float64Col(pos=6)
time_max = Float64Col(pos=7)
state_size_med = Int32Col(pos=8)
state_size_max = Int32Col(pos=9)
def _save_row_spec(self):
descriptor = {
'end_row': Int32Col(),
'header_rows':
StringCol(itemsize=255), # comma separated ints or empty string.
'start_row': Int32Col(),
'comment_rows':
StringCol(itemsize=255), # comma separated ints or empty string.
'data_pattern': StringCol(itemsize=255)
}
self._save_meta_child('row_spec', descriptor)
class _FeatureDescription(IsDescription):
seqid = StringCol(256)
start = Int32Col()
end = Int32Col()
Name = StringCol(256)
biotype = StringCol(256)
class ChromosomeTable(tables.IsDescription): """ 2013.3.8 """ id = UInt64Col(pos=0) name = StringCol(512, pos=1) #should be unique, but not 100% enforced #name should look like species_name.chromosome_name, if species_name is available length = UInt64Col(pos=2) sex_chromosome = StringCol(4, pos=3) #A=autosome, X=chr X, Y=chr Y species_id = UInt64Col(pos=4) path = StringCol(4512, pos=5) #path to the file containing the chromosome sequences, should be unique,
class PolymorphismTable(tables.IsDescription): """ 2013.3.6 table that records the polymorphism alleles of individuals at relevant loci """ id = UInt64Col(pos=0) name = StringCol(512, pos=1) #name should be "individualName.locusName.chromosome_copy" to ensure uniqueness individual_id = UInt64Col(pos=2) locus_id = UInt64Col(pos=3) chromosome_copy = UInt64Col(pos=4, dflt=0) #starting from 0 allele_sequence = StringCol(512, pos=5) allele_sequence_length = UInt64Col(pos=6) allele_type = StringCol(512, pos=7)
class AssociationLocusTable(tables.IsDescription): """ 2013.1.28 bugfix, was pos=3 for no_of_peaks (same as stop), now change it to pos=4, and increment others accordingly """ id = UInt64Col(pos=0) chromosome = StringCol(64, pos=1) #64 byte-long start = UInt64Col(pos=2) stop = UInt64Col(pos=3) no_of_peaks = UInt64Col(pos=4) connectivity = Float64Col(pos=5) no_of_results = UInt64Col(pos=6) phenotype_id_ls_in_str = StringCol(1000, pos=7)
class CountAssociationLocusTable(tables.IsDescription):
"""
2012.12.24 new PyTables-based table definition
"""
id = UInt64Col(pos=0)
min_score = Float64Col(pos=1)
min_overlap_ratio = Float64Col(pos=2)
total_no_of_results = UInt64Col(pos=3)
no_of_association_loci = UInt64Col(pos=4)
call_method_id_ls = StringCol(1000, pos=5)
cnv_method_id_ls = StringCol(1000, pos=6)
phenotype_method_id_ls = StringCol(1000, pos=7)
analysis_method_id_ls = StringCol(1000, pos=8)
class Preopen(IsDescription):
symbol = StringCol(50)
xDt = StringCol(50)
caAct = StringCol(50)
iep = Float32Col()
chn = Float16Col()
perChn = Float16Col()
pCls = Float32Col()
trdQnty = Int32Col()
iVal = Float16Col()
sumVal = Float32Col()
sumQnty = Int32Col()
finQnty = Int32Col()
sumfinQnty = Int32Col()
class Equity(IsDescription):
SYMBOL = StringCol(50)
SERIES = StringCol(2)
OPEN = Float32Col()
HIGH = Float32Col()
LOW = Float32Col()
CLOSE = Float32Col()
LAST = Float32Col()
PREVCLOSE = Float32Col()
TOTTRDQTY = Int32Col()
TOTTRDVAL = Float64Col()
TIMESTAMP = StringCol(12)
TOTALTRADES = Int32Col()
ISIN = StringCol(12)
def saveField(h5, resultGroup, result):
def dump(inputList):
def conversion(arg):
if type(arg) == type(u' '):
return arg.encode('utf-8')
else:
return arg.__repr__()
if type(inputList) == type([]):
return map(conversion, inputList)
else:
return map(dump, inputList)
if result.data.dtype.char in ['U', 'O']:
unicodeData = scipy.array(dump(result.data.tolist()))
h5.createArray(resultGroup, "data", unicodeData,
result.longname.encode("utf-8"))
else:
h5.createArray(resultGroup, "data", result.data,
result.longname.encode("utf-8"))
for key, value in result.attributes.iteritems():
h5.setNodeAttr(resultGroup.data, key, value)
h5.setNodeAttr(resultGroup, "longname", result.longname.encode("utf-8"))
h5.setNodeAttr(resultGroup, "shortname", result.shortname.encode("utf-8"))
h5.setNodeAttr(resultGroup, "creator", result.creator.encode("utf-8"))
h5.setNodeAttr(resultGroup, "machine", result.machine.encode("utf-8"))
if result.error != None:
h5.createArray(resultGroup, "error", result.error,
(u"Error of " + result.longname).encode("utf-8"))
if result.mask != None:
h5.createArray(resultGroup, "mask", result.mask,
(u"Mask of " + result.longname).encode("utf-8"))
h5.setNodeAttr(resultGroup, "unit", repr(result.unit).encode("utf-8"))
if result.dimensions != DataContainer.INDEX:
idLen = max([len(dim.id.encode("utf-8")) for dim in result.dimensions])
dimTable = h5.createTable(resultGroup,
"dimensions", {
"hash": StringCol(32),
"id": StringCol(idLen)
}, (u"Dimensions of " +
result.longname).encode("utf-8"),
expectedrows=len(result.dimensions))
for dim in result.dimensions:
d = dimTable.row
d["hash"] = dim.hash.encode("utf-8")
d["id"] = dim.id.encode("utf-8")
d.append()
saveResult(dim, h5)
dimTable.flush()
class Derivative(IsDescription):
INSTRUMENT = StringCol(6)
SYMBOL = StringCol(50)
EXPIRY_DT = StringCol(11)
STRIKE_PR = Int32Col()
OPTION_TYP = StringCol(2)
OPEN = Float32Col()
HIGH = Float32Col()
LOW = Float32Col()
CLOSE = Float32Col()
SETTLE_PR = Float32Col()
CONTRACTS = Int32Col()
VAL_INLAKH = Float64Col()
OPEN_INT = Int32Col()
CHG_IN_OI = Int32Col()
TIMESTAMP = StringCol(11)
class OXTS(IsDescription):
"""HFS5 OXTS table model"""
timestamp = StringCol(32)
lat = Float64Col()
lon = Float64Col()
alt = Float64Col()
roll = Float64Col()
pitch = Float64Col()
yaw = Float64Col()
vn = Float64Col()
ve = Float64Col()
vf = Float64Col()
vl = Float64Col()
vu = Float64Col()
ax = Float64Col()
ay = Float64Col()
az = Float64Col()
af = Float64Col()
al = Float64Col()
au = Float64Col()
wx = Float64Col()
wy = Float64Col()
wz = Float64Col()
wf = Float64Col()
wl = Float64Col()
wu = Float64Col()
pos_accuracy = Float64Col()
vel_accuracy = Float64Col()
navstat = UInt8Col()
numsats = UInt8Col()
posmode = UInt8Col()
velmode = UInt8Col()
orimode = UInt8Col()
class Array(IsDescription):
"""Row descriptor class for Pytables array data table.
Contains parameter information for each selected telescope
in the data.
Attributes
----------
tel_id : UInt8Col
UInt8 placeholder type for the telescope id (in the array)
tel_x : Float32Col
Float32 placeholder type for the telescope position x coordinate
relative to the center of the array.
tel_y : Float32Col
Float32 placeholder type for the telescope position y coordinate
relative to the center of the array.
tel_z : Float32Col
Float32 placeholder type for the telescope position z coordinate
(height) relative to the CORSIKA observatory altitude.
tel_type : StringCol
String placeholder type for the telescope type name (i.e. 'LST')
run_array_direction: Float32Col(2)
Float32 tuple placeholder type for the array pointing direction for
a given run (az,alt)
"""
tel_id = UInt8Col()
tel_x = Float32Col()
tel_y = Float32Col()
tel_z = Float32Col()
tel_type = StringCol(8)
run_array_direction = Float32Col(2)
def __init__(self, p, collength=16, verbose=False):
"""
Instantiates a MAP object corresponding to MAP files.
The data_format is how the file is set up for use with reading and writing HDF5 tables. The dictionary
key represents the name of the column, the first part of the value tuple represents where it is located
when the line is split and the second part represents the datatype to use within the table.
:param p: The full file path
"""
self.data_format = {
"chromosome": (0, StringCol(collength)),
"identifier": (1, StringCol(collength)),
"distance": (2, StringCol(collength)),
"position": (3, StringCol(collength))
}
DataType.__init__(self, p, "map", verbose)
class ArrayTableRow(IsDescription):
"""Describe row format for telescope array table.
Contains parameter information for each telescope in the array.
NOTE: Additional columns are added dynamically to some tables, see the
github wiki page for the full table/data format descriptions.
Attributes
----------
id : tables.UInt8Col
Telescope id (unique).
type : tables.StringCol
Telescope type name (i.e. 'LST:LSTCam').
x : tables.Float32Col
Telescope position x coordinate relative to the center of the array.
y : tables.Float32Col
Telescope position y coordinate relative to the center of the array.
z : tables.Float32Col
Telescope position z coordinate (height) relative to the CORSIKA
observatory altitude.
"""
id = UInt16Col()
type = StringCol(20)
x = Float32Col()
y = Float32Col()
z = Float32Col()
class PopulationTable(tables.IsDescription): """ 2013.3.8 """ id = UInt64Col(pos=0) name = StringCol(512, pos=1) #name should like 'species_name.population_name' size = UInt64Col(pos=2) species_id = UInt64Col(pos=3)
def test_converts_comment_rows_and_header_rows_json_to_list(self):
temp_fs = fsopendir('temp://')
# save meta.row_spec to the file.
descriptor = { # this is not valid descriptor, but I do not need it to be valid here.
'header_rows': StringCol(itemsize=255),
'comment_rows': StringCol(itemsize=255),
}
self._write_test_meta(temp_fs, 'row_spec', descriptor,
[[('comment_rows', json.dumps([0, 1])),
('header_rows', json.dumps([2, 3]))]])
# now read it from file.
with open_file(temp_fs.getsyspath('temp.h5'), 'r') as h5:
ret = HDFReader._read_meta_child(h5, 'row_spec')
self.assertEqual(ret[0]['comment_rows'], [0, 1])
self.assertEqual(ret[0]['header_rows'], [2, 3])
def test_converts_hist_and_uvalues_json_to_list(self):
temp_fs = fsopendir('temp://')
# save meta.schema minimal table to the file.
descriptor = { # this is not valid descriptor, but I do not need it to be valid here.
'hist': StringCol(itemsize=255),
'uvalues': StringCol(itemsize=255),
}
self._write_test_meta(temp_fs, 'schema', descriptor,
[[('hist', json.dumps([0, 1])),
('uvalues', json.dumps(['a', 'b']))]])
# now read it from file.
with open_file(temp_fs.getsyspath('temp.h5'), 'r') as h5:
ret = HDFReader._read_meta_child(h5, 'schema')
self.assertEqual(ret[0]['hist'], [0, 1])
self.assertEqual(ret[0]['uvalues'], ['a', 'b'])
class CameraScanTableDescription(IsDescription):
"""
"""
setpoint = Float32Col()
frame_path = StringCol(140)
ravg = Float32Col()
gavg = Float32Col()
bavg = Float32Col()
