def create_input_group(h5f,
title="input data at transmitter",
rolloff_dflt=np.nan,
attrs={},
arrays=["symbols", "bits"],
**kwargs):
"""
Create the table for saving the input symbols and bits
Parameters
----------
h5f : string or h5filehandle
The file to use, if a string create or open new file
title: string, optional
The title description of the group
attrs: dict, optional
attributes on the table
arrays: list, optional
name of arrays referenced in the table
**kwargs:
keyword arguments passed to create_table/array, it is highly recommended to set expectedrows
Returns
-------
h5f : h5filehandle
Pytables handle to the hdf file
"""
try:
gr = h5f.create_group("/", "input", title=title)
except AttributeError:
h5f = tb.open_file(h5f, "a")
gr = h5f.create_group("/", "input", title=title)
# if no shape for input syms or bits is given use scalar
t_in = h5f.create_table(gr,
"signal", {
"id": tb.Int64Col(),
"idx_symbols": tb.Int64Col(dflt=0),
"idx_bits": tb.Int64Col(dflt=0),
"rolloff": tb.Float64Col(dflt=rolloff_dflt)
},
title="parameters of input signal",
**kwargs)
setattr(t_in.attrs, "arrays", arrays)
arr_syms = h5f.create_mdvlarray(gr,
"symbols",
tb.ComplexAtom(itemsize=16, dflt=np.nan),
title="sent symbols",
**kwargs)
arr_bits = h5f.create_mdvlarray(gr,
"bits",
tb.BoolAtom(),
title="sent bits",
**kwargs)
for k, v in attrs:
setattr(t_in.attrs, k, v)
return h5f
class ResultRecord(tables.IsDescription):
"""
Kruskal Wallis
"""
chromosome = tables.Int32Col()
position = tables.Int32Col()
score = tables.Float64Col()
maf = tables.Float32Col()
mac = tables.Int32Col()
class info(tb.IsDescription):
_v_pos = 1
name = tb.StringCol(10)
value = tb.Float64Col(pos=0)
y2 = tb.Float64Col(dflt=1, shape=(2, 3), pos=1)
z2 = tb.UInt8Col(dflt=1)
class info2(tb.IsDescription):
y3 = tb.Float64Col(dflt=1, shape=(2, 3))
z3 = tb.UInt8Col(dflt=1)
name = tb.StringCol(10)
value = tb.EnumCol(colors, 'blue', base='uint32', shape=(1,))
class info3(tb.IsDescription):
name = tb.StringCol(10)
value = tb.Time64Col()
y4 = tb.Float64Col(dflt=1, shape=(2, 3))
z4 = tb.UInt8Col(dflt=1)
class LightningConfig(tables.IsDescription):
event_id = tables.UInt32Col(pos=0)
timestamp = tables.Time32Col(pos=1)
com_port = tables.UInt8Col(pos=2)
baud_rate = tables.Int16Col(pos=3)
station_id = tables.UInt32Col(pos=4)
database_name = tables.Int32Col(dflt=-1, pos=5)
help_url = tables.Int32Col(dflt=-1, pos=6)
daq_mode = tables.BoolCol(pos=7)
latitude = tables.Float64Col(pos=8)
longitude = tables.Float64Col(pos=9)
altitude = tables.Float64Col(pos=10)
squelch_seting = tables.Int32Col(pos=11)
close_alarm_distance = tables.Int32Col(pos=12)
severe_alarm_distance = tables.Int32Col(pos=13)
noise_beep = tables.BoolCol(pos=14)
minimum_gps_speed = tables.Int32Col(pos=15)
angle_correction = tables.Float32Col(pos=16)
class DataSensor(tb.IsDescription):
"""Store metadata information for the SiPMs (position, gain,
calibration-constant, mask).
"""
channel = tb.Int32Col(pos=0) # electronic channel
position = tb.Float32Col(pos=1, shape=3)
coeff = tb.Float64Col(pos=2)
adc_to_pes = tb.Float32Col(pos=3)
noise_rms = tb.Float32Col(pos=4)
class MCHitInfo(tb.IsDescription):
"""Stores the simulated hits as metadata using Pytables.
"""
hit_position = tb.Float32Col(pos=0, shape=3)
hit_time = tb.Float64Col(pos=1)
hit_energy = tb.Float32Col(pos=2)
label = tb.StringCol(20, pos=3)
particle_indx = tb.Int16Col(pos=4)
hit_indx = tb.Int16Col(pos=5)
class Region(tables.IsDescription):
file_key = tables.UInt32Col(pos=0)
chromosome = tables.StringCol(nps.chromosome_name_length, pos=1)
region_name = tables.StringCol(64, pos=2)
start = tables.UInt64Col(pos=3)
stop = tables.UInt64Col(pos=4)
strand = tables.StringCol(1, pos=5)
count = tables.UInt64Col(pos=6)
normalized_count = tables.Float64Col(pos=7)
class Particle(tb.IsDescription):
name = tb.StringCol(16) # 16-character String
idnumber = tb.Int64Col() # Signed 64-bit integer
ADCcount = tb.UInt16Col() # Unsigned short integer
TDCcount = tb.UInt8Col() # unsigned byte
grid_i = tb.Int32Col() # 32-bit integer
grid_j = tb.Int32Col() # 32-bit integer
pressure = tb.Float32Col() # float (single-precision)
energy = tb.Float64Col() # double (double-precision)
class Trade(tables.IsDescription):
time = tables.Int64Col()
buy_order_id = tables.Int64Col()
sell_order_id = tables.Int64Col()
buyer = tables.Int64Col()
seller = tables.Int64Col()
price = tables.Float64Col()
quantity = tables.Int64Col()
side = tables.Int8Col()
class Particle(tables.IsDescription):
ADCcount = tables.Int16Col() # signed short integer
TDCcount = tables.UInt8Col() # unsigned byte
grid_i = tables.Int32Col() # integer
grid_j = tables.Int32Col() # integer
idnumber = tables.Int64Col() # signed long long
name = tables.StringCol(16, dflt="") # 16-character String
pressure = tables.Float32Col(shape=2) # float (single-precision)
temperature = tables.Float64Col() # double (double-precision)
class Coords_and_vel(tables.IsDescription):
time = tables.Float64Col()
x_pos = tables.Float64Col()
vx = tables.Float64Col()
y_pos = tables.Float64Col()
vy = tables.Float64Col()
z_pos = tables.Float64Col()
vz = tables.Float64Col()
class TruthData(tables.IsDescription):
E = tables.Float64Col(pos=0)
x = tables.Float64Col(pos=1)
y = tables.Float64Col(pos=2)
z = tables.Float64Col(pos=3)
px = tables.Float64Col(pos=4)
py = tables.Float64Col(pos=5)
pz = tables.Float64Col(pos=6)
class Timecheck(tables.IsDescription):
time = tables.Float64Col()
x = tables.Float64Col()
vx = tables.Float64Col()
y = tables.Float64Col()
vy = tables.Float64Col()
z = tables.Float64Col()
vz = tables.Float64Col()
class info2(tables.IsDescription):
y3 = tables.Float64Col(dflt=1, shape=(2, 3))
z3 = tables.UInt8Col(dflt=1)
name = tables.StringCol(10)
value = tables.EnumCol(colors, 'blue', base='uint32', shape=(1,))
class info3(tables.IsDescription):
name = tables.StringCol(10)
value = tables.Time64Col()
y4 = tables.Float64Col(dflt=1, shape=(2, 3))
z4 = tables.UInt8Col(dflt=1)
class HitsTable(tb.IsDescription):
event = tb. Int32Col(pos=0)
time = tb.Float64Col(pos=1)
npeak = tb. UInt16Col(pos=2)
nsipm = tb. UInt16Col(pos=3)
X = tb.Float64Col(pos=4)
Y = tb.Float64Col(pos=5)
Xrms = tb.Float64Col(pos=6)
Yrms = tb.Float64Col(pos=7)
Z = tb.Float64Col(pos=8)
Q = tb.Float64Col(pos=9)
E = tb.Float64Col(pos=10)
class Create(tb.IsDescription):
nrows = tb.Int32Col(pos=0)
irows = tb.Int32Col(pos=1)
tfill = tb.Float64Col(pos=2)
tidx = tb.Float64Col(pos=3)
tcfill = tb.Float64Col(pos=4)
tcidx = tb.Float64Col(pos=5)
rowsecf = tb.Float64Col(pos=6)
rowseci = tb.Float64Col(pos=7)
fsize = tb.Float64Col(pos=8)
isize = tb.Float64Col(pos=9)
psyco = tb.BoolCol(pos=10)
class MonitoringSubarrayPointing(tables.IsDescription):
"""
Create the r0 Subarray pointing inside the monitoring directory
"""
time = tables.Float64Col()
tels_with_trigger = tables.UInt64Col()
array_azimuth = tables.Float32Col()
array_altitude = tables.Float32Col()
array_ra = tables.Float32Col()
array_dec = tables.Float32Col()
class _TableBody(tables.IsDescription):
name = tables.StringCol(20)
additional_names = tables.StringCol(20)
body_type = tables.StringCol(5)
constellation = tables.StringCol(4)
ra = tables.Float64Col()
dec = tables.Float64Col()
mag = tables.Float64Col()
surface_brightness = tables.StringCol(4)
size_max = tables.StringCol(8)
size_min = tables.StringCol(8)
positional_angle = tables.Float64Col()
sci_class = tables.StringCol(11)
central_star_mag = tables.Float64Col()
catalog = tables.StringCol(4)
ngc_descr = tables.StringCol(55)
notes = tables.StringCol(86)
class ResultRecordFT(tables.IsDescription):
"""
Fisher's exact test
"""
chromosome = tables.Int32Col()
position = tables.Int32Col()
score = tables.Float64Col()
maf = tables.Float32Col()
mac = tables.Int32Col()
odds_ratio = tables.Float32Col()
class HistrogramData(tables.IsDescription):
energy = tables.Float64Col()
number = tables.Int32Col()
bins_low = tables.Float64Col(shape=bins_1.size)
bins_high = tables.Float64Col(shape=bins_2.size)
gamma_hist_low = tables.Float64Col(shape=bins_1.size - 1)
gamma_hist_high = tables.Float64Col(shape=bins_2.size - 1)
electron_hist_low = tables.Float64Col(shape=bins_1.size - 1)
electron_hist_high = tables.Float64Col(shape=bins_2.size - 1)
class Trades(tables.IsDescription):
time = tables.Int64Col()
trader_id = tables.Int64Col()
trade_id = tables.Int64Col()
sequence_id = tables.Int64Col()
side = tables.Int8Col()
price = tables.Float64Col()
quantity = tables.Int64Col()
origin_id = tables.Int8Col()
is_auction = tables.BoolCol()
is_aggressor = tables.BoolCol()
class _PointDescriptor(tables.IsDescription):
""" Descriptor for storing Point information
Provides the column definition to store Point
information (x, y, z).
"""
uid = tables.StringCol(32, pos=0)
data = tables.StringCol(32, pos=1)
coordinates = tables.Float64Col(pos=2, shape=(3, ))
class Info(t.IsDescription): #1
_v_pos = 1
name = t.StringCol(itemsize=2)
value = t.ComplexCol(itemsize=16, pos=0) #0
y2 = t.Float64Col(dflt=1, pos=1) #1
z2 = t.UInt8Col(dflt=1)
class Info2(t.IsDescription):
y3 = t.Time64Col(dflt=1, shape=2)
z3 = t.EnumCol({'r':4, 'g':2, 'b':1}, 'r', 'int32', shape=2)
name = t.StringCol(itemsize=2)
value = t.ComplexCol(itemsize=16, shape=2)
def main():
vector_size = 128
with tables.open_file('/tmp/8GB.h5', 'w') as f:
table = f.create_table(f.root, 'table', {
'vector_values':
tables.Float64Col(pos=0, shape=(vector_size, )),
})
vector = np.zeros(vector_size)
for _ in range(8300000):
table.row['vector_values'] = vector
table.row.append()
def __init__(self, output_filename, n_permutations):
self.h5file = tables.open_file(output_filename, 'w')
self.column_names = ['snp_id'] + [
'permutation_' + str(x) for x in range(n_permutations)
]
#define the permutation result object on-the-fly, depending on the number of permutations that will be performed
self.permutation_result_definition = dict([
(x, tables.Float64Col()) for x in self.column_names
if x.split('_')[0] == 'permutation'
])
self.permutation_result_definition['snp_id'] = tables.StringCol(100)
class PLS_temp(tb.IsDescription):
"""PLS table row template"""
PLS_id = tb.Int16Col(dflt=0, pos=0)
g1 = tb.Float64Col(dflt=0.0, pos=1)
resopfrac = tb.Float64Col(dflt=0.0, pos=2)
tleaf = tb.Float32Col(dflt=0.0, pos=3)
twood = tb.Float32Col(dflt=0.0, pos=4)
troot = tb.Float32Col(dflt=0.0, pos=5)
aleaf = tb.Float32Col(dflt=0.0, pos=6)
awood = tb.Float32Col(dflt=0.0, pos=7)
aroot = tb.Float32Col(dflt=0.0, pos=8)
c4 = tb.UInt8Col(dflt=0, pos=9)
leaf_n2c = tb.Float32Col(dflt=0.0, pos=10)
awood_n2c = tb.Float32Col(dflt=0.0, pos=11)
froot_n2c = tb.Float32Col(dflt=0.0, pos=12)
leaf_p2c = tb.Float32Col(dflt=0.0, pos=13)
awood_p2c = tb.Float32Col(dflt=0.0, pos=14)
froot_p2c = tb.Float32Col(dflt=0.0, pos=15)
amp = tb.Float32Col(dflt=0.0, pos=16)
pdia = tb.Float32Col(dflt=0.0, pos=17)
class Distribution(tb.IsDescription):
"""
kind of volume distribution
Table structure:
- date : POSIX 시간(초)을 Integer 형태로 저장
- value : 거래량 / (고가 - 저가)
- price : 각 value의 column index
"""
row = tb.UInt64Col(pos=0)
price = tb.Float64Col(pos=1)
value = tb.Float16Col(pos=2)
class MetaTable(tb.IsDescription):
index_start = tb.UInt32Col(pos=0)
# start_index
index_stop = tb.UInt32Col(pos=1)
# stop_index
data_length = tb.UInt32Col(pos=2)
# length = tb.UInt32Col(pos=2)
# https://github.com/PyTables/PyTables/issues/230
#timestamp = tb.Time64Col(pos=3)
timestamp = tb.Float64Col(pos=3)
error = tb.UInt32Col(pos=4)
def create_meas_group(h5f,
title="measurement data",
description=None,
attrs=MEAS_UNITS,
arrays=["data"],
**kwargs):
"""
Create the table for saving oscilloscope measurements
Parameters
----------
h5f : string or h5filehandle
The file to use, if a string create or open new file
title: string, optional
The title description of the group
data_shape: int
Number of modes/polarizations
description: dict or tables.IsDescription (optional)
If given use to create the table
arrays: list, optional
name of arrays referenced in the table
attrs: dict, optional
attributes on the table
**kwargs:
keyword arguments passed to create_table/array, it is highly recommended to set expectedrows
Returns
-------
h5f : h5filehandle
Pytables handle to the hdf file
"""
try:
gr_meas = h5f.create_group("/", "measurements", title=title)
except AttributeError:
h5f = tb.open_file(h5f, "a")
gr_meas = h5f.create_group("/", "measurements", title=title)
gr_osc = h5f.create_group(gr_meas,
"oscilloscope",
title="Data from Realtime oscilloscope")
if description is None:
description = {
"id": tb.Int64Col(),
"samplingrate": tb.Float64Col(),
"idx_data": tb.Int64Col()
}
t_meas = h5f.create_table(gr_osc, "signal", description, "sampled signal",
**kwargs)
setattr(t_meas.attrs, "arrays", arrays)
arr = h5f.create_mdvlarray(gr_osc, "data", tb.ComplexAtom(itemsize=16),
**kwargs)
for k, v in attrs.items():
setattr(t_meas.attrs, k, v)
return h5f
def create_recvd_data_group(h5f, title="data analysis and qampy results", description=None, oversampling_dflt=2,
attrs=DSP_UNITS, arrays=["data", "symbols", "taps", "bits"], nmodes=2, **kwargs):
"""
Create the table for saving recovered data and parameters after DSP
Parameters
----------
h5f : string or h5filehandle
The file to use, if a string create or open new file
title: string
The title description of the group
description: dict or tables.IsDescription (optional)
If given use to create the table
attrs: dict, optional
attributes for the table
arrays: list, optional
name of arrays referenced in the table
nmodes: int, optional
number of modes/polarisations
**kwargs:
keyword arguments passed to create_table/array, it is highly recommended to set expectedrows
Returns
-------
h5f : h5filehandle
Pytables handle to the hdf file
"""
try:
gr = h5f.create_group("/", "analysis", title=title)
except AttributeError:
h5f = tb.open_file(h5f, "a")
gr = h5f.create_group("/", "analysis", title=title)
gr_dsp = h5f.create_group(gr, "qampy", title="Signal from DSP")
if description is None:
dsp_params = { "freq_offset": tb.Float64Col(dflt=np.nan),
"freq_offset_N": tb.Int64Col(dflt=0), "phase_est": tb.StringCol(itemsize=20),
"N_angles": tb.Float64Col(dflt=np.nan), "ph_est_blocklength": tb.Int64Col(),
"stepsize": tb.Float64Col(shape=2), "trsyms": tb.Float64Col(shape=2),
"iterations": tb.Int64Col(shape=2),
"ntaps": tb.Int64Col(),
"method": tb.StringCol(itemsize=20)}
description = {"id":tb.Int64Col(), "idx_data": tb.Int64Col(), "idx_symbols": tb.Int64Col(),
"idx_bits": tb.Int64Col(), "idx_taps": tb.Int64Col(),
"evm": tb.Float64Col(dflt=np.nan, shape=nmodes), "ber":tb.Float64Col(dflt=np.nan, shape=nmodes),
"ser":tb.Float64Col(dflt=np.nan, shape=nmodes), "oversampling":tb.Int64Col(dflt=oversampling_dflt)}
description.update(dsp_params)
t_rec = h5f.create_table(gr_dsp, "signal", description, "signal after DSP", **kwargs)
setattr(t_rec.attrs, "arrays", arrays)
data_arr = h5f.create_mdvlarray(gr_dsp, "data", tb.ComplexAtom(itemsize=16), "signal after DSP", **kwargs)
syms_arr = h5f.create_mdvlarray(gr_dsp, "symbols", tb.ComplexAtom(itemsize=16, dflt=np.nan), "recovered symbols", **kwargs)
taps_arr = h5f.create_mdvlarray(gr_dsp, "taps", tb.ComplexAtom(itemsize=16, dflt=np.nan), "qampy taps", **kwargs)
bits_arr = h5f.create_mdvlarray(gr_dsp, "bits", tb.BoolAtom(dflt=False), "recovered bits", **kwargs)
for k, v in attrs.items():
setattr(t_rec.attrs, k, v)
return h5f
