class OrderModel(pt.IsDescription):
task = pt.StringCol(5)
shares = pt.Int32Col()
symbol = pt.StringCol(30)
order_type = pt.StringCol(5) #moo moc limit vwap
duration = pt.Time64Col()
timestamp = pt.Time64Col()
close_type = pt.StringCol(4) #lifo or fifo for a sell, none for a buy
limit_price = pt.Float32Col()
fill = FillModel()
class StrategyDataModel(pt.IsDescription):
symbol = pt.StringCol(30) #30 char string; Ticker
exchange = pt.StringCol(10) #10 char string; NYSE, NASDAQ, etc.
adj_high = pt.Float32Col()
adj_low = pt.Float32Col()
adj_open = pt.Float32Col()
adj_close = pt.Float32Col()
close = pt.Float32Col()
volume = pt.Int32Col()
timestamp = pt.Time64Col()
date = pt.Int32Col()
interval = pt.Time64Col()
def __init__(self, h5file: tables.File, parent_group: tables.Group,
exp_id: str, exp_title: str, variables: Mapping[str,
VarType]):
super(_ExperimentWriter, self).__init__()
self._id = exp_id
self._title = exp_title
self._file = h5file
try:
self._group = h5file.create_group(parent_group,
exp_id,
title=exp_title)
except tables.NodeError:
try:
node = h5file.get_node(parent_group, exp_id)
path = node._v_pathname
if isinstance(node, tables.Group):
raise ExperimentElementError('Experiment already exists at'
f'{path} in file {h5file}.')
elif isinstance(node, tables.Table):
raise ExperimentElementError('Name conflict: variable '
'table already exists at '
f'{path} in file {h5file}')
else:
raise ExperimentElementError(f'Conflict at {path} '
f'in file {h5file}')
except tables.NoSuchNodeError as e:
raise ExperimentElementError() from e
# metadata
self._group._v_attrs.created = datetime.datetime.now().isoformat()
self._group._v_attrs.finished = 'unfinished'
self._var_tables = dict()
for var_name, var_type in variables.items():
try:
tbl = h5file.create_table(self._group,
var_name,
description={
'record_time':
tables.Time64Col(),
'experiment_time':
tables.Time64Col(),
'value':
_vartype_columns[var_type]()
})
self._var_tables[var_name] = tbl
except KeyError:
raise UnsupportedVariableType(var_type)
self._sub_experiments = dict()
class ScanPointing(tb.IsDescription):
"""PyTables table descriptor: pointing details for scans"""
timestamp = tb.Time64Col(pos=0) # Timestamp at telescope pointing
mb01_raj = tb.Float32Col(pos=1) # Beam 01 RA (degrees)
mb01_dcj = tb.Float32Col(pos=2) # Beam 01 DEC (degrees)
mb02_raj = tb.Float32Col(pos=3) # Beam 02 RA
mb02_dcj = tb.Float32Col(pos=4) # Beam 02 DEC
mb03_raj = tb.Float32Col(pos=5) # .
mb03_dcj = tb.Float32Col(pos=6) # .
mb04_raj = tb.Float32Col(pos=7) # .
mb04_dcj = tb.Float32Col(pos=8)
mb05_raj = tb.Float32Col(pos=9)
mb05_dcj = tb.Float32Col(pos=10)
mb06_raj = tb.Float32Col(pos=11)
mb06_dcj = tb.Float32Col(pos=12)
mb07_raj = tb.Float32Col(pos=13)
mb07_dcj = tb.Float32Col(pos=14)
mb08_raj = tb.Float32Col(pos=15)
mb08_dcj = tb.Float32Col(pos=16)
mb09_raj = tb.Float32Col(pos=17)
mb09_dcj = tb.Float32Col(pos=18)
mb10_raj = tb.Float32Col(pos=19)
mb10_dcj = tb.Float32Col(pos=20)
mb11_raj = tb.Float32Col(pos=21)
mb11_dcj = tb.Float32Col(pos=22)
mb12_raj = tb.Float32Col(pos=23)
mb12_dcj = tb.Float32Col(pos=24)
mb13_raj = tb.Float32Col(pos=25)
mb13_dcj = tb.Float32Col(pos=26)
azimuth = tb.Float32Col(pos=27) # Telescope azimuth
elevation = tb.Float32Col(pos=28) # Telescope elevation
par_angle = tb.Float32Col(pos=29) # Paraxial angle
focus_tan = tb.Float32Col(pos=30) # Lateral focus offset (mm) "Z"
focus_axi = tb.Float32Col(pos=31) # Axial docus offset (mm) "Y"
focus_rot = tb.Float32Col(pos=32) # Receiver rotation angle
def reset_table():
"""Creates h5f flatfile for storing price data
Returns
-------
h5f handle : returns a writable file handle for pytables table
"""
h5f = tables.openFile('price_data.h5', 'w')
description = {
"ticker": tables.StringCol(itemsize=6, dflt='', pos=1),
"frequency": tables.StringCol(itemsize=1, dflt='d', pos=2),
"date": tables.Time32Col(dflt=0.00, pos=3),
"open": tables.Float32Col(dflt=0.00, pos=4),
"high": tables.Float32Col(dflt=0.00, pos=5),
"low": tables.Float32Col(dflt=0.00, pos=6),
"close": tables.Float32Col(dflt=0.00, pos=7),
"volume": tables.Int64Col(dflt=0.00, pos=8),
"adjustedClose": tables.Float32Col(dflt=0.00, pos=9),
"timestamp": tables.Time64Col(dflt=0.00, pos=10)
}
table = h5f.createTable('/', 'price_data', description)
h5f.close()
def pytables_desc(self):
"""
Convert NumPy dtype to PyTable descriptor (lifted from blaze.pytables).
Examples
--------
>>> from tables import Int32Col, StringCol, Time64Col
>>> dt = np.dtype([('name', 'S7'), ('amount', 'i4'), ('time', 'M8[us]')])
>>> dtype_to_pytables(dt) # doctest: +SKIP
{'amount': Int32Col(shape=(), dflt=0, pos=1),
'name': StringCol(itemsize=7, shape=(), dflt='', pos=0),
'time': Time64Col(shape=(), dflt=0.0, pos=2)}
"""
dtype = np.dtype(self.target_dtype)
d = {}
for pos, name in enumerate(dtype.names):
dt, _ = dtype.fields[name]
if issubclass(dt.type, np.datetime64):
tdtype = tb.defscription({name: tb.Time64Col(pos=pos)}),
else:
tdtype = tb.descr_from_dtype(np.dtype([(name, dt)]))
el = tdtype[0] # removed dependency on toolz -DJC
getattr(el, name)._v_pos = pos
d.update(el._v_colobjects)
return d
class OilProductionByStateAndMonth(tables.IsDescription):
"""Data model class for oil production by state and month/year"""
date = tables.Time64Col()
la_barrels = tables.Int32Col()
tx_barrels = tables.Int32Col()
ak_barrels = tables.Int32Col()
ca_barrels = tables.Int32Col()
class Weather(tb.IsDescription):
"""PyTables table descriptor: weather details"""
timestamp = tb.Time64Col(pos=0) # Timestamp at telescope pointing
temperature = tb.Float32Col(pos=1) # Ambient temperature (K)
pressure = tb.Float32Col(pos=2) # Pressure (kPa)
humidity = tb.Float32Col(pos=3) # Humidity (%)
wind_speed = tb.Float32Col(pos=4) # Wind Speed (m/s)
wind_direction = tb.Float32Col(pos=5) # Wind Direction (deg)
class BaseSignal(tb.IsDescription):
"""
Base class for signals
"""
name = tb.UInt64Col()
message = tb.StringCol(128)
protocol = tb.StringCol(32)
incoming_time = tb.Time64Col()
class AlphaDataModelClass(pt.IsDescription):
symbol = pt.StringCol(30)
exchange = pt.StringCol(10)
alphaValue = pt.Float32Col()
timestamp = pt.Time64Col()
def __init__(self):
print("In the AlphaDataModelClass constructor")
class Spectrum_mbcal(tb.IsDescription):
""" PyTables table descriptor: storage of spectral data
For hipsr_mbcal modes. Only cal information is stored.
"""
id = tb.Int32Col(pos=0) # Unique ID
timestamp = tb.Time64Col(pos=1) # Timestamp (at BRAM read)
xx_cal_on = tb.Int32Col(shape=16, pos=6) # Noise diode ON, X pol
xx_cal_off = tb.Int32Col(shape=16, pos=7) # Noise diode OFF, X pol
yy_cal_on = tb.Int32Col(shape=16, pos=8) # Noise diode ON, Y pol
yy_cal_off = tb.Int32Col(shape=16, pos=9) # Noise diode OFF, Y pol
class IndexMsg1553(tables.IsDescription):
''' Class to hold index information
msg_index - Array index of message
msg_time - UTC time of indexed message
'''
offset = tables.UInt64Col()
time = tables.Time64Col()
channel_id = tables.UInt16Col()
rt = tables.UInt8Col()
tr = tables.UInt8Col()
subaddr = tables.UInt8Col()
class Particle(tables.IsDescription):
"""This class defines a table record.
"""
lati = tables.IntCol(pos=0)
# longi = IntCol(pos=1)
Time = tables.Time64Col(pos=2)
pressure = tables.FloatCol(pos=3)
ID = tables.StringCol(itemsize=10, pos=4)
Int16 = tables.UIntCol(itemsize=4, pos=5)
Int64 = tables.IntCol(itemsize=8, pos=6)
Bool = tables.BoolCol(pos=7)
def main(filename):
# Identify LDC1614 EVM by USB VID/PID match
detected_ports = list(serial.tools.list_ports.grep('2047:08F8'))
if not detected_ports:
raise RuntimeError('No EVM found.')
else:
# open the serial device.
evm = serial.Serial(detected_ports[0].device, 115200, timeout=1)
device_id = read_reg(evm, LDC1614_DEVICE_ID)
ldc_config(evm)
h5f = tables.open_file(filename, 'a', title="LDC1614EVM Logger Data")
try:
tbl = h5f.get_node('/logdata')
print("Appending existing table in: {}".format(filename))
except tables.NoSuchNodeError:
table_definition = {
'time_utc': tables.Time64Col(),
'data_ch0': tables.UInt32Col(),
'data_ch1': tables.UInt32Col()
}
tbl = h5f.create_table('/',
'logdata',
description=table_definition,
title='LDC1614 dataset')
print("Created new table in: {}".format(filename))
f_ref = 40.0
C = 33.0
L = 18.0
start_stream(evm)
print("Beginning logging...")
while True:
(raw_ch0, raw_ch1, raw_ch2, raw_ch3) = read_stream(evm)
if raw_ch0 and not (raw_ch0 & 0xF0000000):
tbl.row['time_utc'] = time.time()
tbl.row['data_ch0'] = raw_ch3
f_sensor = raw_ch0 * f_ref / 268435456
C_sensor = 1 / L * (2 * np.pi * f_sensor) * (2 * np.pi *
f_sensor) - C
print(C_sensor)
tbl.row['data_ch1'] = raw_ch1
#print(raw_ch1)
tbl.row.append()
tbl.flush()
# If we handled errors like KeyboardInterrupt properly, we'd get here:
print("Cleaning up...")
tbl.close()
h5f.close()
class Spectrum_12_4096(tb.IsDescription):
""" PyTables table descriptor: storage of spectral data
For hipsr_200_16384 firmware: 200 MHz, 16384 channels
"""
id = tb.Int32Col(pos=0) # Unique ID
timestamp = tb.Time64Col(pos=1) # Timestamp (at BRAM read)
xx = tb.UInt32Col(shape=4096, pos=2) # XX Autocorrelation data
yy = tb.UInt32Col(shape=4096, pos=3) # YY Autocorrelation data
# re_xy = tb.Int32Col(shape=16384,pos=4) # XY Cross correlation - real
#im_xy = tb.Int32Col(shape=16384,pos=5) # XY Cross correlation - imag
xx_cal_on = tb.Int32Col(shape=16, pos=6) # Noise diode ON, X pol
xx_cal_off = tb.Int32Col(shape=16, pos=7) # Noise diode OFF, X pol
yy_cal_on = tb.Int32Col(shape=16, pos=8) # Noise diode ON, Y pol
yy_cal_off = tb.Int32Col(shape=16, pos=9) # Noise diode OFF, Y pol
fft_of = tb.BoolCol(pos=10) # FFT overflow flag
adc_clip = tb.BoolCol(pos=11) # ADC clipping flag
class Observation(tb.IsDescription):
"""PyTables table descriptor: observation details"""
telescope = tb.StringCol(32, pos=0) # Telescope name (always Parkes for us)
receiver = tb.StringCol(32, pos=1) # Receiver name (MULTI for us)
date = tb.Time64Col(pos=2) # Date - only 32bits reqd for date but using 64
project_id = tb.StringCol(32, pos=3) # Project ID number, PXXX for Parkes
project_name = tb.StringCol(255, pos=4) # Project name
observer = tb.StringCol(255, pos=5) # Observer's name
num_beams = tb.Int8Col(pos=6) # Number of beams being used
ref_beam = tb.Int8Col(pos=7) # Reference beam
acc_len = tb.Float32Col(pos=8) # Accumulation length, in seconds
bandwidth = tb.Int32Col(pos=9) # Bandwidth (MHz) (-ve means inverted)
dwell_time = tb.Float32Col(pos=10) # Dwell time (sec)
frequency = tb.Float32Col(pos=11) # Central frequency (MHz)
feed_rotation = tb.StringCol(64, pos=12) # Feed rotation (e.g. STEPPED)
feed_angle = tb.Float32Col(pos=13) # Feed angle
freq_switch = tb.BoolCol(pos=14) # Frequency switching flag
obs_mode = tb.StringCol(16, pos=15) # Observation mode (e.g SCAN)
scan_rate = tb.Float32Col(pos=16) # Scan rate, (deg/min)
def dtype_to_pytables(dtype):
""" Convert NumPy dtype to PyTable descriptor
Examples
--------
>>> from tables import Int32Col, StringCol, Time64Col
>>> dt = np.dtype([('name', 'S7'), ('amount', 'i4'), ('time', 'M8[us]')])
>>> dtype_to_pytables(dt) # doctest: +SKIP
{'amount': Int32Col(shape=(), dflt=0, pos=1),
'name': StringCol(itemsize=7, shape=(), dflt='', pos=0),
'time': Time64Col(shape=(), dflt=0.0, pos=2)}
"""
d = {}
for pos, name in enumerate(dtype.names):
dt, _ = dtype.fields[name]
if issubclass(dt.type, np.datetime64):
tdtype = tb.Description({name: tb.Time64Col(pos=pos)}),
else:
tdtype = tb.descr_from_dtype(np.dtype([(name, dt)]))
el = first(tdtype)
getattr(el, name)._v_pos = pos
d.update(el._v_colobjects)
return d
def dtype_to_table(dtype):
""" Convert a NumPy dtype to a PyTables Table description
Essentially just :ref:`tables.descr_from_dtype` but it works on
:ref:`np.datetime64`
Args:
dtype (np.dtype): NumPy data type
Returns:
dict: PyTables description
"""
desc = {}
for idx, name in enumerate(dtype.names):
_dt, _ = dtype.fields[name]
if issubclass(_dt.type, np.datetime64):
tb_dtype = tb.Description({name: tb.Time64Col(pos=idx)})
else:
tb_dtype, byteorder = tb.descr_from_dtype(np.dtype([(name, _dt)]))
_tb_dtype = tb_dtype._v_colobjects
_tb_dtype[name]._v_pos = idx
desc.update(_tb_dtype)
return desc
class SpoolEvent(tables.IsDescription):
EventName = tables.StringCol(32)
Time = tables.Time64Col()
EventDescr = tables.StringCol(256)
class Info2(t.IsDescription):
y3 = t.Time64Col(dflt=1, shape=2)
z3 = t.EnumCol({'r':4, 'g':2, 'b':1}, 'r', 'int32', shape=2)
name = t.StringCol(itemsize=2)
value = t.ComplexCol(itemsize=16, shape=2)
class TS(tables.IsDescription):
timestamp = tables.Time64Col(pos=0)
ticker = tables.Float64Col(shape=(10, ))
trades = tables.Float64Col(shape=(120, 4))
book = tables.Float64Col(shape=(50, 3))
class MyTimeRow(tb.IsDescription):
t32col = tb.Time32Col(shape=(2, 1))
t64col = tb.Time64Col(shape=(2, 1))
class MyTimeRow(tb.IsDescription):
i8col = tb.Int8Col(pos=0)
t32col = tb.Time32Col(pos=1)
t64col = tb.Time64Col(shape=(2, ), pos=2)
class PositionModel(pt.IsDescription):
timestamp = pt.Time64Col()
symbol = pt.StringCol(30)
shares = pt.Int32Col()
purchase_price = pt.Float32Col()
class MyTimeRow(tables.IsDescription):
intcol = tables.IntCol(shape=(2, 1))
t32col = tables.Time32Col(shape=(2, 1))
t64col = tables.Time64Col(shape=(2, 1))
class MyTimeRow(tables.IsDescription):
t32col = tables.Time32Col(pos=0)
t64col = tables.Time64Col(shape=(2, ), pos=1)
class TimingTableConfig(tb.IsDescription):
time = tb.Time64Col()
class MyTimeRow(tb.IsDescription):
intcol = tb.IntCol()
t32col = tb.Time32Col()
t64col = tb.Time64Col()
class SpoolEvent(tables.IsDescription):
"""Pytables description for Events table in spooled dataset"""
EventName = tables.StringCol(32)
Time = tables.Time64Col()
EventDescr = tables.StringCol(256)
class info3(tables.IsDescription):
name = tables.StringCol(10)
value = tables.Time64Col()
y4 = tables.Float64Col(dflt=1, shape=(2, 3))
z4 = tables.UInt8Col(dflt=1)
