class AgentGameOutcomeRow(IsDescription):
game_number = Int32Col(pos=0)
opponent = StringCol(name_size, pos=1)
version = Int32Col(pos=2)
moved_first = BoolCol(pos=3)
outcome = StringCol(outcome_size, pos=4)
when = TimeStamp()
def _create_h5(filename):
# save meta.about to the file.
with open_file(filename, 'w') as h5:
descriptor1 = {
'field1': Float64Col(),
'field2': Float64Col(),
'field3': Float64Col()
}
descriptor2 = {
'version': Int32Col(),
'n_rows': Int32Col(),
'n_cols': Int32Col()
}
h5.create_group('/partition', 'meta', createparents=True)
rows_table = h5.create_table('/partition', 'rows', descriptor1)
file_header_table = h5.create_table('/partition', 'file_header',
descriptor2)
for i in range(5):
rows_table.row['field1'] = float(i)
rows_table.row['field2'] = float(i)
rows_table.row['field2'] = float(i)
rows_table.row.append()
rows_table.flush()
file_header_table.row['version'] = 1
file_header_table.row['n_rows'] = 1
file_header_table.row['n_cols'] = 1
file_header_table.row.append()
file_header_table.flush()
class AgentVersionRow(IsDescription):
version = Int32Col(pos=0, dflt=0)
won = Int32Col(pos=1, dflt=0)
lost = Int32Col(pos=2, dflt=0)
drawn = Int32Col(pos=3, dflt=0)
failed = Int32Col(pos=4, dflt=0)
rating = Float64Col(pos=5, dflt=0.0)
uploaded = TimeStamp()
class Particle(IsDescription):
name = StringCol(16) # 16-character String
idnumber = Int64Col() # Signed 64-bit integer
ADCcount = UInt16Col() # Unsigned short integer
TDCcount = UInt8Col() # unsigned byte
grid_i = Int32Col() # 32-bit integer
grid_j = Int32Col() # 32-bit integer
pressure = Float32Col() # float (single-precision)
energy = Float64Col() # double (double-precision)
class GameSummaryAgent(IsDescription):
name = StringCol(name_size, pos=0)
version = Int32Col(pos=1)
game_number = Int32Col(pos=2)
rating = Float64Col(pos=3, dflt=0.0)
outcome = StringCol(outcome_size, pos=4)
total_time = Float64Col(pos=5)
time_med = Float64Col(pos=6)
time_max = Float64Col(pos=7)
state_size_med = Int32Col(pos=8)
state_size_max = Int32Col(pos=9)
def _save_row_spec(self):
descriptor = {
'end_row': Int32Col(),
'header_rows':
StringCol(itemsize=255), # comma separated ints or empty string.
'start_row': Int32Col(),
'comment_rows':
StringCol(itemsize=255), # comma separated ints or empty string.
'data_pattern': StringCol(itemsize=255)
}
self._save_meta_child('row_spec', descriptor)
class _FeatureDescription(IsDescription):
seqid = StringCol(256)
start = Int32Col()
end = Int32Col()
Name = StringCol(256)
biotype = StringCol(256)
class McmcParameters(IsDescription):
"""
Class that holds the data model for the HDF5 table with the MCMC parameters.
"""
iterations = Int32Col()
burnIn = Int32Col()
thin = Int32Col()
minMeanAbsoluteMagnitude = Float64Col()
maxMeanAbsoluteMagnitude = Float64Col()
priorTau = StringCol(40)
shapeTau = Float64Col()
scaleTau = Float64Col()
class Post(IsDescription):
id = Int32Col()
post_type_id = UInt8Col()
parent_id = Int32Col()
accepted_answer_id = Int32Col()
creation_date = Time32Col()
score = UInt32Col()
viewcount = UInt32Col()
owner_user_id = Int32Col()
answer_count = UInt16Col()
favorite_count = UInt16Col()
tags = StringCol(itemsize=25, shape=(5, ))
class Equity(IsDescription):
SYMBOL = StringCol(50)
SERIES = StringCol(2)
OPEN = Float32Col()
HIGH = Float32Col()
LOW = Float32Col()
CLOSE = Float32Col()
LAST = Float32Col()
PREVCLOSE = Float32Col()
TOTTRDQTY = Int32Col()
TOTTRDVAL = Float64Col()
TIMESTAMP = StringCol(12)
TOTALTRADES = Int32Col()
ISIN = StringCol(12)
class Preopen(IsDescription):
symbol = StringCol(50)
xDt = StringCol(50)
caAct = StringCol(50)
iep = Float32Col()
chn = Float16Col()
perChn = Float16Col()
pCls = Float32Col()
trdQnty = Int32Col()
iVal = Float16Col()
sumVal = Float32Col()
sumQnty = Int32Col()
finQnty = Int32Col()
sumfinQnty = Int32Col()
class Record(tables.IsDescription):
var1 = StringCol(itemsize=4, dflt=b"abcd", pos=0)
var2 = StringCol(itemsize=1, dflt=b"a", pos=1)
var3 = BoolCol(dflt=1)
var4 = Int8Col(dflt=1)
var5 = UInt8Col(dflt=1)
var6 = Int16Col(dflt=1)
var7 = UInt16Col(dflt=1)
var8 = Int32Col(dflt=1)
var9 = UInt32Col(dflt=1)
var10 = Int64Col(dflt=1)
var11 = Float32Col(dflt=1.0)
var12 = Float64Col(dflt=1.0)
var13 = ComplexCol(itemsize=8, dflt=(1. + 0.j))
var14 = ComplexCol(itemsize=16, dflt=(1. + 0.j))
if hasattr(tables, 'Float16Col'):
var15 = tables.Float16Col(dflt=1.0)
if hasattr(tables, 'Float96Col'):
var16 = tables.Float96Col(dflt=1.0)
if hasattr(tables, 'Float128Col'):
var17 = tables.Float128Col(dflt=1.0)
if hasattr(tables, 'Complex196Col'):
var18 = tables.ComplexCol(itemsize=24, dflt=(1. + 0.j))
if hasattr(tables, 'Complex256Col'):
var19 = tables.ComplexCol(itemsize=32, dflt=(1. + 0.j))
class _VariantDescription(IsDescription):
CHROM = StringCol(256)
POS = Int32Col()
ID = StringCol(256)
REF = StringCol(256)
ALT = StringCol(256)
QUAL = Float32Col()
CAF = StringCol(256)
CLNDISDB = StringCol(256)
CLNDN = StringCol(256)
CLNSIG = StringCol(256)
CLNREVSTAT = StringCol(256)
CLNVI = StringCol(256)
effect = StringCol(256)
impact = StringCol(256)
gene_name = StringCol(256)
GT = StringCol(256)
class Test_GrainData(IsDescription):
"""
Description class specifying structured storage for tests
"""
idnumber = Int32Col() # Signed 64-bit integer
volume = Float32Col() # float
center = Float32Col(shape=(3, )) # float
def _get_rows_descriptor(columns):
""" Converts columns specifications from ambry_sources format to pytables descriptor.
Args:
columns (list of dict)
Returns:
dict: valid pytables descriptor.
"""
TYPE_MAP = {
'int': lambda pos: Int32Col(pos=pos),
'long': lambda pos: Int64Col(pos=pos),
'str': lambda pos: StringCol(itemsize=255, pos=pos),
'bytes': lambda pos: StringCol(itemsize=255, pos=pos),
'float': lambda pos: Float64Col(pos=pos),
'unknown': lambda pos: StringCol(itemsize=255, pos=pos),
}
descriptor = {}
for column in columns:
pytables_type = TYPE_MAP.get(column['type'])
if not pytables_type:
raise Exception(
'Failed to convert `{}` ambry_sources type to pytables type.'.
format(column['type']))
descriptor[column['name']] = pytables_type(column['pos'])
return descriptor
class Particle(IsDescription):
particle_id = Int32Col()
mass = Float64Col()
x_pos = Float64Col()
y_pos = Float64Col()
x_vel = Float64Col()
y_vel = Float64Col()
class Derivative(IsDescription):
INSTRUMENT = StringCol(6)
SYMBOL = StringCol(50)
EXPIRY_DT = StringCol(11)
STRIKE_PR = Int32Col()
OPTION_TYP = StringCol(2)
OPEN = Float32Col()
HIGH = Float32Col()
LOW = Float32Col()
CLOSE = Float32Col()
SETTLE_PR = Float32Col()
CONTRACTS = Int32Col()
VAL_INLAKH = Float64Col()
OPEN_INT = Int32Col()
CHG_IN_OI = Int32Col()
TIMESTAMP = StringCol(11)
class User(IsDescription):
id = Int32Col()
name = StringCol(16)
age = UInt8Col()
reputation = UInt16Col()
creation_date = Time32Col()
views = UInt16Col()
upvotes = UInt16Col()
downvotes = UInt16Col()
class SurveyParameters(IsDescription):
"""
Class that holds the data model for the simulated parallax survey parameters. Intended for use with the
HDF5 files through the pytables package.
"""
kind = StringCol(itemsize=40)
numberOfStars = Int32Col()
minParallax = Float64Col()
maxParallax = Float64Col()
meanAbsoluteMagnitude = Float64Col()
varianceAbsoluteMagnitude = Float64Col()
parallaxErrorNormalizationMagnitude = Float64Col()
parallaxErrorSlope = Float64Col()
parallaxErrorCalibrationFloor = Float64Col()
magnitudeErrorNormalizationMagnitude = Float64Col()
magnitudeErrorSlope = Float64Col()
magnitudeErrorCalibrationFloor = Float64Col()
apparentMagnitudeLimit = Float64Col()
numberOfStarsInSurvey = Int32Col()
def test_saves_header_rows_and_comment_rows_as_json_string(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
writer._validate_groups()
writer.meta['row_spec']['comment_rows'] = [0, 1]
writer.meta['row_spec']['header_rows'] = [2, 3]
descriptor = {
'end_row': Int32Col(),
'header_rows': StringCol(itemsize=255),
'start_row': Int32Col(),
'comment_rows': StringCol(itemsize=255),
'data_pattern': StringCol(itemsize=255)
}
writer._save_meta_child('row_spec', descriptor)
self.assertEqual(
[(x['comment_rows'], x['header_rows'])
for x in writer._h5_file.root.partition.meta.row_spec.iterrows()],
[(b('[0, 1]'), b('[2, 3]'))])
def write_file_header(self):
""" Write the version, number of rows and number of cols to the h5 file. """
if 'file_header' in self._h5_file.root.partition:
self._h5_file.remove_node('/partition', 'file_header')
descriptor = {
'version': Int32Col(),
'n_rows': Int32Col(),
'n_cols': Int32Col()
}
table = self._h5_file.create_table('/partition', 'file_header',
descriptor, 'Header of the file.')
table.row['version'] = HDFPartition.VERSION
table.row['n_rows'] = self.n_rows
table.row['n_cols'] = self.n_cols
table.row.append()
table.flush()
class SSelectorOnMeshElement(IsDescription):
shortName = StringCol(ELEMENT_LENGTH, pos=0)
imin = Int32Col(pos=1)
jmin = Int32Col(pos=2)
kmin = Int32Col(pos=3)
imax = Int32Col(pos=4)
jmax = Int32Col(pos=5)
kmax = Int32Col(pos=6)
v1 = Float32Col()
v2 = Float32Col()
v3 = Float32Col()
def setup_spec_table(self):
if not hasattr(self.node, "ms1_spec_table"):
for level in (1, 2):
description = {}
description["pm_index"] = UInt32Col(pos=0)
description["rt"] = Float32Col(pos=1)
description["scan_number"] = Int32Col(pos=2)
description["start"] = UInt64Col(pos=3)
description["end"] = UInt64Col(pos=4)
t = self.file_.create_table(self.node, 'ms%d_spec_table' % level, description,
filters=filters)
# every colums which appears in a where method call should/must be indexed !
# this is not only for performance but for correct lookup as well (I had strange bugs
# else)
t.cols.pm_index.create_index()
t.cols.rt.create_index()
class TestTDescr(tables.IsDescription):
"""A description that has several nested columns."""
x = Int32Col(dflt=0, shape=2, pos=0) # 0
y = FloatCol(dflt=1, shape=(2, 2))
z = UInt8Col(dflt=1)
z3 = EnumCol({'r': 4, 'g': 2, 'b': 1}, 'r', 'int32', shape=2)
color = StringCol(itemsize=4, dflt=b"ab", pos=2)
info = Info()
class Info(tables.IsDescription): # 1
_v_pos = 1
name = StringCol(itemsize=2)
value = ComplexCol(itemsize=16, pos=0) # 0
y2 = FloatCol(pos=1) # 1
z2 = UInt8Col()
class Info2(tables.IsDescription):
y3 = Time64Col(shape=2)
name = StringCol(itemsize=2)
value = ComplexCol(itemsize=16, shape=2)
class _VariantDescription(IsDescription):
"""
Describe VariantHDF5 table columns.
"""
# TODO: Optimize
CHROM = StringCol(8)
POS = Int32Col()
ID = StringCol(16)
REF = StringCol(256)
ALT = StringCol(256)
QUAL = Float32Col()
# INFO
CAF = StringCol(16)
CLNSIG = StringCol(8)
CLNDBN = StringCol(256)
# INFO ANN
effect = StringCol(32)
impact = StringCol(32)
gene_name = StringCol(32)
# FORMAT & sample
GT = StringCol(8)
class description(IsDescription):
time = Int32Col()
st = Int8Col(shape=(N_CH, ))
wave = Float32Col(shape=(S_TOTAL, N_CH))
fet = Float32Col(shape=(N_CH, FPC))
clu = Int32Col()
class Video(IsDescription):
label = Int32Col()
frames = Float32Col(shape=(29, 112, 112))
yaw = Float32Col()
file = StringCol(32)
word = StringCol(32)
class ParametersTableRow(IsDescription):
"""Describe row format for parameter table.
Contains parameters values for each image of each event of each telescope in the array.
Parameters are calculated after image cleaning (i.e. with for example tailcut_clean method)
There are Hillas, Leakage, Concentration, Timing and Morphology parameters.
Attributes
----------
event_index : tables.Int32Col
Event id of file (from -1 to N )
leakage_* : tables.Float32Col
see at https://cta-observatory.github.io/ctapipe/api/ctapipe.containers.LeakageContainer.html?highlight=leakagecontainer#ctapipe.containers.LeakageContainer
hillas_* : tables.Float32Col
see at https://cta-observatory.github.io/ctapipe/api/ctapipe.containers.HillasParametersContainer.html#ctapipe.containers.HillasParametersContainer
concentration_* :
see at https://cta-observatory.github.io/ctapipe/api/ctapipe.containers.ConcentrationContainer.html#ctapipe.containers.ConcentrationContainer
timing_* :
see at https://cta-observatory.github.io/ctapipe/api/ctapipe.containers.TimingParametersContainer.html#ctapipe.containers.TimingParametersContainer
morphology_* :
see at https://cta-observatory.github.io/ctapipe/api/ctapipe.containers.MorphologyContainer.html#ctapipe.containers.MorphologyContainer
log_hillas_intensity : tables.Float32Col
logaritmic hillas intensity
"""
event_index = Int32Col()
leakage_intensity_width_1 = Float32Col()
leakage_intensity_width_2 = Float32Col()
leakage_pixels_width_1 = Float32Col()
leakage_pixels_width_2 = Float32Col()
hillas_intensity = Float32Col()
log_hillas_intensity = Float32Col()
hillas_x = Float32Col()
hillas_y = Float32Col()
hillas_r = Float32Col()
hillas_phi = Float32Col()
hillas_width = Float32Col()
hillas_length = Float32Col()
hillas_psi = Float32Col()
hillas_skewness = Float32Col()
hillas_kurtosis = Float32Col()
concentration_cog = Float32Col()
concentration_core = Float32Col()
concentration_pixel = Float32Col()
timing_slope = Float32Col() # time gradient
timing_slope_err = Float32Col()
timing_intercept = Float32Col()
timing_intercept_err = Float32Col()
timing_deviation = Float32Col()
morphology_num_pixels = Int32Col()
morphology_num_islands = Int32Col()
morphology_num_small_islands = Int32Col()
morphology_num_medium_islands = Int32Col()
morphology_num_large_islands = Int32Col()
class Event(IsDescription):
id = Int32Col()
timestamp = Time32Col()
event_type = UInt8Col()
event_id = Int32Col()
class Vote(IsDescription):
id = Int32Col()
post_id = Int32Col()
vote_type_id = UInt8Col()
creation_date = Time32Col()
