class Data2DCameraSummary(PT.IsDescription):
cam_id = PT.StringCol(16, pos=0)
camn = PT.Int32Col(pos=1)
start_frame = PT.UInt64Col(pos=2)
stop_frame = PT.UInt64Col(pos=3)
start_timestamp = PT.FloatCol(pos=4)
stop_timestamp = PT.FloatCol(pos=5)
class ExactMovieInfo(PT.IsDescription):
cam_id = PT.StringCol(16, pos=0)
filename = PT.StringCol(255, pos=1)
start_frame = PT.UInt64Col(pos=2)
stop_frame = PT.UInt64Col(pos=3)
start_timestamp = PT.FloatCol(pos=4)
stop_timestamp = PT.FloatCol(pos=5)
def init_cluster_table(self):
if self.read_mode == 'w':
try:
self.db_main.remove_node('/', 'cluster_table')
print 'Cluster table dropped.'
except tables.exceptions.NoSuchNodeError:
print 'cluster table does not exist yet. nothing to drop.'
try:
cluster_desc = {
'train_uuid': tables.StringCol(32),
'ticker': tables.StringCol(10),
'kpi': tables.StringCol(15),
'c0': tables.FloatCol(),
'c1': tables.FloatCol(),
'c2': tables.FloatCol(),
'c3': tables.FloatCol(),
'c4': tables.FloatCol()
}
self.t_clusters = self.db_main.create_table(
'/', 'cluster_table', cluster_desc)
self.t_clusters.cols.ticker.create_index()
self.t_clusters.cols.kpi.create_index()
print 'cluster table created.'
except tables.exceptions.NodeError:
self.t_clusters = self.db_main.get_node('/', 'cluster_table')
print 'cluster table exists already.'
else:
self.t_clusters = self.db_main.get_node('/', 'cluster_table')
print 'cluster table exists already.'
class DirectionReconstruction(tb.IsDescription):
MC_Energy = tb.FloatCol(dflt=1, pos=1)
N_LST = tb.Int16Col(dflt=1, pos=2)
N_MST = tb.Int16Col(dflt=1, pos=3)
N_SST = tb.Int16Col(dflt=1, pos=4)
multiplicity = tb.Int64Col(dflt=1, pos=5)
off_angle = tb.FloatCol(dflt=1, pos=6)
offset = tb.FloatCol(dflt=1, pos=7)
class VectorDescription(pytables.IsDescription):
"""
Customized record for writing solution vectors including a counter
and time information.
"""
counter = pytables.Int64Col() # Signed 64-bit integer
values = pytables.FloatCol(shape=u_1.vector().array().shape) #
time = pytables.FloatCol() # double (double-precision)
def init_recommendation_table(self):
if self.read_mode == 'w':
try:
self.db_main.remove_node('/', 'recommend')
print 'recommendation table dropped.'
except tables.exceptions.NoSuchNodeError:
print 'recommendation table does not exist yet. nothing to initialize.'
try:
recommend_desc = {
'sim_uuid': tables.StringCol(32),
'dix': tables.IntCol(),
'symbol': tables.StringCol(10),
'tradeTx': tables.StringCol(10),
'tradePriceMin': tables.FloatCol(),
'tradePriceMid': tables.FloatCol(),
'tradePriceMax': tables.FloatCol(),
'trade12dd': tables.IntCol(),
'tradeIdxPct': tables.FloatCol(),
'tradeDate': tables.StringCol(10),
'tradeDateCopy': tables.StringCol(10),
'offsetTraining': tables.IntCol(),
'minTraining': tables.IntCol(),
'modelAccuracy': tables.FloatCol()
}
self.t_recommend = self.db_main.create_table(
'/', 'recommend', recommend_desc)
self.t_recommend.cols.sim_uuid.create_index()
self.t_recommend.cols.dix.create_index()
print 'recommendation table created.'
except tables.exceptions.NodeError:
self.t_recommend = self.db_main.get_node('/', 'recommend')
print 'recommendation table opened.'
else:
try:
self.t_recommend = self.db_main.get_node('/', 'recommend')
print 'recommendation table opened.'
except tables.exceptions.NoSuchNodeError:
recommend_desc = {
'sim_uuid': tables.StringCol(32),
'dix': tables.IntCol(),
'symbol': tables.StringCol(10),
'tradeTx': tables.StringCol(10),
'tradePriceMin': tables.FloatCol(),
'tradePriceMid': tables.FloatCol(),
'tradePriceMax': tables.FloatCol(),
'trade12dd': tables.IntCol(),
'tradeIdxPct': tables.FloatCol(),
'tradeDate': tables.StringCol(10),
'tradeDateCopy': tables.StringCol(10),
'offsetTraining': tables.IntCol(),
'minTraining': tables.IntCol(),
'modelAccuracy': tables.FloatCol()
}
self.t_recommend = self.db_main.create_table(
'/', 'recommend', recommend_desc)
self.t_recommend.cols.sim_uuid.create_index()
self.t_recommend.cols.dix.create_index()
class Particle(tables.IsDescription):
"""Description of a table record."""
name = tables.StringCol(16, pos=1)
lati = tables.IntCol(pos=2)
vector = tables.Int32Col(shape=(200, ), pos=3)
matrix1 = tables.Int32Col(shape=(2, 200), pos=4)
matrix2 = tables.FloatCol(shape=(100, 2), pos=5)
matrix3 = tables.FloatCol(shape=(10, 100, 2), pos=5)
matrix4 = tables.FloatCol(shape=(2, 10, 100, 2), pos=5)
class TimeEncodedSignalDescriptor(t.IsDescription):
"""Descriptor of saved time-encoded signal."""
comment = t.StringCol(64, pos=1) # description of signal
bw = t.FloatCol(pos=2, dflt=1.0) # bandwidth (rad/s)
dt = t.FloatCol(pos=3, dflt=1.0) # interval between samples (s)
b = t.FloatCol(pos=4, dflt=1.0) # bias
d = t.FloatCol(pos=5, dflt=1.0) # threshold
k = t.FloatCol(pos=6, dflt=1.0) # integration constant
class ReconstructedKascadeEvent(tables.IsDescription):
"""Store information about reconstructed events"""
# r, phi is core position
id = tables.UInt32Col()
station_id = tables.UInt8Col()
r = tables.Float32Col()
phi = tables.Float32Col()
alpha = tables.Float32Col()
t1 = tables.Float32Col()
t2 = tables.Float32Col()
t3 = tables.Float32Col()
t4 = tables.Float32Col()
n1 = tables.Float32Col()
n2 = tables.Float32Col()
n3 = tables.Float32Col()
n4 = tables.Float32Col()
reference_theta = tables.Float32Col()
reference_phi = tables.Float32Col()
reconstructed_theta = tables.Float32Col()
reconstructed_phi = tables.Float32Col()
min_n134 = tables.Float32Col()
k_energy = tables.FloatCol()
k_core_pos = tables.FloatCol(shape=2)
k_Num_e = tables.FloatCol()
k_Num_mu = tables.FloatCol()
k_dens_e = tables.FloatCol(shape=4)
k_dens_mu = tables.FloatCol(shape=4)
k_P200 = tables.FloatCol()
k_T200 = tables.FloatCol()
class _Event(tb.IsDescription):
"""
Description of the table /events/eventTable.
"""
# UIntAtom = uint32
array_row = tb.UIntCol(pos=0) # indicates the corresponding row in the
event_start = tb.UIntCol(itemsize=8,
pos=1) # start index of the event in the data
event_length = tb.UIntCol(pos=2)
n_levels = tb.UIntCol(pos=3)
raw_points_per_side = tb.UIntCol(pos=4)
baseline = tb.FloatCol(pos=5)
current_blockage = tb.FloatCol(pos=6)
area = tb.FloatCol(pos=7)
def connect_PyTable(self,
filepath,
table_name='example',
h5f_description=None):
self.h5f = tb.open_file(filepath, 'w')
self.h5f_table_name = table_name
if h5f_description is None:
tmp_desc = {
'coordX': tb.FloatCol(),
'coordY': tb.FloatCol(),
'APN': tb.StringCol(12),
'maz': tb.Int32Col()
}
tbl = self.h5f.create_table('/', self.h5f_table_name, tmp_desc)
self.USING_PYTABLES = True
def init_q_table(self):
if self.read_mode == 'w':
try:
self.db_main.remove_node('/', 'q_table')
print 'q_table table dropped.'
except tables.exceptions.NoSuchNodeError:
print 'no q table to drop.'
try:
q_table_desc = {
'ticker': tables.IntCol(),
'state': tables.IntCol(),
'action': tables.IntCol(),
'reward': tables.FloatCol()
}
self.t_q = self.db_main.create_table('/', 'q_table',
q_table_desc)
self.t_q.cols.ticker.create_index()
self.t_q.cols.state.create_index()
self.t_q.cols.action.create_index()
print 'q table created.'
except tables.exceptions.NodeError:
self.t_q = self.db_main.get_node('/', 'q_table')
print 'q table opened.'
else:
self.t_q = self.db_main.get_node('/', 'q_table')
print 'q table opened.'
def init_stats_table(self):
if self.read_mode == 'w':
try:
self.db_main.remove_node('/', 'stats')
print 'stats table dropped.'
except tables.exceptions.NoSuchNodeError:
print 'stats table not exist yet. nothing to initialize.'
try:
stats_desc = {
'train_uuid': tables.StringCol(32),
'pca': tables.IntCol(),
'ticker': tables.StringCol(10),
'model': tables.StringCol(3),
'kpi': tables.StringCol(15),
'accuracy': tables.FloatCol()
}
self.t_stats = self.db_main.create_table(
'/', 'stats', stats_desc)
self.t_stats.cols.ticker.create_index()
self.t_stats.cols.kpi.create_index()
print 'statistics table created.'
except tables.exceptions.NodeError:
self.t_stats = self.db_main.get_node('/', 'stats')
print 'stats table opened.'
else:
self.t_stats = self.db_main.get_node('/', 'stats')
print 'stats table opened.'
def _col_descriptor(self):
descriptor = OrderedDict([(k, t.FloatCol(dflt=nan, pos=i))
for i, k in enumerate(self.col_names)])
descriptor['stat'] = t.IntCol(dflt=-1,
pos=self.col_names.index('stat'))
descriptor['qn'] = t.IntCol(dflt=-1, pos=self.col_names.index('qn'))
return descriptor
def init_q_log(self):
if self.read_mode == 'w':
try:
self.db_main.remove_node('/', 'q_log')
print 'stats table dropped.'
except tables.exceptions.NoSuchNodeError:
print 'no q_log to drop.'
try:
q_log_desc = {
'sim_uuid': tables.StringCol(32),
'ticker': tables.StringCol(10),
'dix': tables.IntCol(),
'state': tables.Int64Col(),
'action': tables.IntCol(),
'reward': tables.FloatCol()
}
self.q_log = self.db_main.create_table('/', 'q_log',
q_log_desc)
self.q_log.cols.ticker.create_index()
self.q_log.cols.state.create_index()
self.q_log.cols.action.create_index()
print 'q log table created.'
except tables.exceptions.NodeError:
self.q_log = self.db_main.get_node('/', 'q_log')
print 'q_log table opened.'
else:
self.q_log = self.db_main.get_node('/', 'q_log')
print 'q_log table opened.'
def quickforcesave(fileName, pixels, dataList, description=''):
with tb.open_file(fileName, mode='w', title=description) as h5file:
pix = np.array([[i[0], i[1]] for i in pixels])
parray = h5file.create_array('/', 'center_pix', pix,
'the pixels on which sphere is centered')
parray.attrs.plotting_note = 'remember x and y coordinates are switched when plotting on an image'
dgroup = h5file.create_group(
h5file.root, 'calcData',
'numerical data calculated from measurements at each separation')
for i in dataList:
tablestyle = {}
for j in dataList[i]:
if j == 'Title':
Title = dataList[i][j]
continue
tablestyle[j] = tb.FloatCol()
newtable = h5file.create_table(dgroup, i, tablestyle, title=Title)
aRow = newtable.row
for m, j in enumerate(dataList[i]['separations']):
for k in tablestyle:
#print(j)
aRow[k] = dataList[i][k][m]
aRow.append()
newtable.flush()
return 0
class TimeDelta(tables.IsDescription):
"""Store time differences"""
ext_timestamp = tables.UInt64Col(pos=0)
timestamp = tables.UInt32Col(pos=1)
nanoseconds = tables.UInt32Col(pos=2)
delta = tables.FloatCol(pos=3)
class AccessRecord(tables.IsDescription):
"""
structure for records of file access in associated h5 file
"""
timestamp = tables.FloatCol() # timestamp of operation
computerid = tables.StringCol(36) # name of machine performing op
datastoreid = tables.StringCol(36) # type of action taken
action = tables.StringCol(36) # type of action taken
class Particle(tb.IsDescription):
name = tb.StringCol(16) # 16-character String
idnumber = tb.Int64Col() # Signed 64-bit integer
ADCcount = tb.UInt16Col() # Unsigned short integer
TDCcount = tb.UInt8Col() # Unsigned byte
grid_i = tb.Int32Col() # Integer
grid_j = tb.IntCol() # Integer (equivalent to Int32Col)
pressure = tb.Float32Col() # Float (single-precision)
energy = tb.FloatCol() # Double (double-precision)
class KascadeEvent(tables.IsDescription):
"""Store events from KASCADE"""
run_id = tables.IntCol()
event_id = tables.Int64Col()
timestamp = tables.Time32Col()
nanoseconds = tables.UInt32Col()
ext_timestamp = tables.UInt64Col()
energy = tables.FloatCol()
core_pos = tables.FloatCol(shape=2)
zenith = tables.FloatCol()
azimuth = tables.FloatCol()
Num_e = tables.FloatCol()
Num_mu = tables.FloatCol()
dens_e = tables.FloatCol(shape=4)
dens_mu = tables.FloatCol(shape=4)
P200 = tables.FloatCol()
T200 = tables.FloatCol()
class TSystemSave(tables.IsDescription):
u = tables.FloatCol(shape=u_shape)
mesh = tables.FloatCol(shape=mesh_shape)
lambd = tables.FloatCol(shape=lambd_shape)
dt = tables.FloatCol()
Dt = tables.FloatCol()
eps = tables.FloatCol()
dx = tables.FloatCol()
class KascadeEvent(tables.IsDescription):
"""Store events from KASCADE"""
run_id = tables.IntCol(pos=0)
event_id = tables.Int64Col(pos=1)
timestamp = tables.Time32Col(pos=2)
nanoseconds = tables.UInt32Col(pos=3)
ext_timestamp = tables.UInt64Col(pos=4)
energy = tables.FloatCol(pos=5)
core_pos = tables.FloatCol(pos=6, shape=2)
zenith = tables.FloatCol(pos=7)
azimuth = tables.FloatCol(pos=8)
Num_e = tables.FloatCol(pos=9)
Num_mu = tables.FloatCol(pos=10)
dens_e = tables.FloatCol(pos=11, shape=4)
dens_mu = tables.FloatCol(pos=12, shape=4)
P200 = tables.FloatCol(pos=13)
T200 = tables.FloatCol(pos=14)
class Info2D(PT.IsDescription):
camn = PT.UInt16Col(pos=0)
frame = PT.Int64Col(pos=1)
timestamp = PT.FloatCol(
pos=2
) # when the image trigger happened (returned by timestamp modeler on MainBrain)
cam_received_timestamp = PT.FloatCol(
pos=3
) # when the image was acquired by flydra software (on camera computer)
x = PT.Float32Col(pos=4)
y = PT.Float32Col(pos=5)
area = PT.Float32Col(pos=6)
slope = PT.Float32Col(pos=7)
eccentricity = PT.Float32Col(pos=8)
frame_pt_idx = PT.UInt8Col(
pos=9) # index of point if there were > 1 points in frame
cur_val = PT.UInt8Col(pos=10)
mean_val = PT.Float32Col(pos=11)
sumsqf_val = PT.Float32Col(pos=12) # estimate of <x^2> (running_sumsqf)
def init_simulation_log(self):
try:
perf_desc = {
'sim_uuid': tables.StringCol(32),
'gamma': tables.FloatCol(),
'alpha': tables.FloatCol(),
'simrun': tables.IntCol(),
'dix': tables.IntCol(),
'index': tables.StringCol(10),
'p_value': tables.FloatCol(),
'cash': tables.FloatCol(),
'i_value': tables.IntCol()
}
self.s_log = self.db_main.create_table('/', 's_log', perf_desc)
self.s_log.cols.dix.create_index(kind='full')
print 'simulation log table created.'
except tables.exceptions.NodeError:
s_log = self.db_main.get_node('/', 's_log')
print 'simulation log opened.'
class Info(tb.IsDescription): # 1
_v_pos = 1
name = tb.StringCol(itemsize=2)
value = tb.ComplexCol(itemsize=16, pos=0) # 0
y2 = tb.FloatCol(pos=1) # 1
z2 = tb.UInt8Col()
class Info2(tb.IsDescription):
y3 = tb.Time64Col(shape=2)
name = tb.StringCol(itemsize=2)
value = tb.ComplexCol(itemsize=16, shape=2)
class Particle(tables.IsDescription):
"""This class defines a table record.
"""
lati = tables.IntCol(pos=0)
# longi = IntCol(pos=1)
Time = tables.Time64Col(pos=2)
pressure = tables.FloatCol(pos=3)
ID = tables.StringCol(itemsize=10, pos=4)
Int16 = tables.UIntCol(itemsize=4, pos=5)
Int64 = tables.IntCol(itemsize=8, pos=6)
Bool = tables.BoolCol(pos=7)
class ENMetadata(tables.IsDescription):
artist = tables.StringCol(MAXSTRLEN)
bitrate = tables.IntCol()
duration = tables.FloatCol()
genre = tables.StringCol(MAXSTRLEN)
id = tables.StringCol(MAXSTRLEN)
md5 = tables.StringCol(32)
release = tables.StringCol(MAXSTRLEN)
samplerate = tables.IntCol()
status = tables.StringCol(MAXSTRLEN)
title = tables.StringCol(MAXSTRLEN)
analysispath = tables.StringCol(MAXSTRLEN)
def init_transaction_log(self):
try:
t_log_desc = {
'sim_uuid': tables.StringCol(32),
'dix': tables.IntCol(),
'ticker': tables.StringCol(10),
'tx': tables.StringCol(10),
'price': tables.FloatCol(),
'volume': tables.IntCol(),
'close': tables.FloatCol(),
'cash_before': tables.FloatCol(),
'cash_after': tables.FloatCol(),
'12dd': tables.IntCol()
}
self.t_log = self.db_main.create_table('/', 't_log', t_log_desc)
self.t_log.cols.sim_uuid.create_index()
self.t_log.cols.ticker.create_index()
print 'transaction log table created.'
except tables.exceptions.NodeError:
self.t_log = self.db_main.get_node('/', 't_log')
print 'transaction log opened.'
class ObservationDescriptor(tb.IsDescription):
id = tb.Int64Col(pos=0)
# pwp
pwp_id = tb.Int64Col(pos=1)
pwp_order = tb.Int64Col(pos=2)
ra = tb.FloatCol(pos=3)
dec = tb.FloatCol(pos=4)
mag = tb.FloatCol(pos=5)
mag_err = tb.FloatCol(pos=6)
hjd = tb.FloatCol(pos=7)
chip_nro = tb.FloatCol(pos=8)
stel_cls = tb.FloatCol(pos=9)
elip = tb.FloatCol(pos=10)
pos_ang = tb.FloatCol(pos=11)
# ms
tile_id = tb.Int64Col(pos=12)
source_id = tb.Int64Col(pos=13)
class sami_master(tables.IsDescription):
name = tables.StringCol(19,pos=1)
RA = tables.FloatCol(pos=2)
Dec = tables.FloatCol(pos=3)
r_petro = tables.Float32Col(pos=4)
r_auto = tables.Float32Col(pos=5)
z_tonry = tables.Float32Col(pos=6)
z_spec = tables.Float32Col(pos=7)
M_r = tables.Float32Col(pos=8)
r_e = tables.Float32Col(pos=9)
med_mu_re = tables.Float32Col(pos=10)
mu_re = tables.Float32Col(pos=11)
mu_2re = tables.Float32Col(pos=12)
ellip = tables.Float32Col(pos=13)
PA = tables.Float32Col(pos=14)
Mstar = tables.Float32Col(pos=15)
g_minus_i = tables.Float32Col(pos=16)
A_g = tables.Float32Col(pos=17)
CATID = tables.Int64Col(pos=0)
SURV_SAMI = tables.UInt16Col(pos=18)
PRI_SAMI = tables.UInt16Col(pos=19)
BAD_CLASS = tables.UInt16Col(pos=20)