def test_view_expression_multiple_views_should_all_clear(self):
table = Table({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]})
view = table.view(expressions=[
'// computed \n "a" + "b"',
])
view2 = table.view(expressions=['// computed2 \n "a" - "b"'])
assert view.schema() == {"a": int, "b": int, "computed": float}
assert view2.schema() == {"a": int, "b": int, "computed2": float}
assert view.to_columns() == {
"a": [1, 2, 3, 4],
"b": [5, 6, 7, 8],
"computed": [6, 8, 10, 12],
}
assert view2.to_columns() == {
"a": [1, 2, 3, 4],
"b": [5, 6, 7, 8],
"computed2": [-4, -4, -4, -4],
}
table.clear()
assert view.schema() == {"a": int, "b": int, "computed": float}
assert view2.schema() == {"a": int, "b": int, "computed2": float}
assert view.to_columns() == {}
assert view2.to_columns() == {}
def test_view_computed_with_row_pivots_clear(self):
table = Table({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]})
view = table.view(
row_pivots=["computed"],
computed_columns=[{
"column": "computed",
"computed_function_name": "+",
"inputs": ["a", "b"],
}],
)
assert view.to_columns() == {
"__ROW_PATH__": [[], [6], [8], [10], [12]],
"a": [10, 1, 2, 3, 4],
"b": [26, 5, 6, 7, 8],
"computed": [36.0, 6.0, 8.0, 10.0, 12.0],
}
table.clear()
assert view.to_columns() == {
"__ROW_PATH__": [[]],
"a": [None],
"b": [None],
"computed": [None],
}
def test_view_computed_multiple_dependents_clear(self):
table = Table({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]})
view = table.view(computed_columns=[
{
"column": "computed",
"computed_function_name": "+",
"inputs": ["a", "b"],
},
{
"column": "final",
"computed_function_name": "pow2",
"inputs": ["computed"],
},
])
assert view.to_columns() == {
"a": [1, 2, 3, 4],
"b": [5, 6, 7, 8],
"computed": [6, 8, 10, 12],
"final": [36, 64, 100, 144],
}
table.clear()
assert view.schema() == {
"a": int,
"b": int,
"computed": float,
"final": float,
}
assert view.to_columns() == {}
def test_view_expression_create_clear(self):
table = Table({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]})
view = table.view(expressions=['// computed \n "a" + "b"'])
assert view.to_columns() == {
"a": [1, 2, 3, 4],
"b": [5, 6, 7, 8],
"computed": [6, 8, 10, 12],
}
table.clear()
assert view.schema() == {"a": int, "b": int, "computed": float}
assert view.to_columns() == {}
def test_view_computed_create_clear(self):
table = Table({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]})
view = table.view(computed_columns=[{
"column": "computed",
"computed_function_name": "+",
"inputs": ["a", "b"],
}])
assert view.to_columns() == {
"a": [1, 2, 3, 4],
"b": [5, 6, 7, 8],
"computed": [6, 8, 10, 12],
}
table.clear()
assert view.schema() == {"a": int, "b": int, "computed": float}
assert view.to_columns() == {}
def test_view_computed_multiple_views_should_all_clear(self):
table = Table({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]})
view = table.view(computed_columns=[{
"column": "computed",
"computed_function_name": "+",
"inputs": ["a", "b"],
}])
view2 = table.view(computed_columns=[{
"column": "computed2",
"computed_function_name": "-",
"inputs": ["a", "b"],
}])
assert view.schema() == {"a": int, "b": int, "computed": float}
assert view2.schema() == {"a": int, "b": int, "computed2": float}
assert view.to_columns() == {
"a": [1, 2, 3, 4],
"b": [5, 6, 7, 8],
"computed": [6, 8, 10, 12],
}
assert view2.to_columns() == {
"a": [1, 2, 3, 4],
"b": [5, 6, 7, 8],
"computed2": [-4, -4, -4, -4],
}
table.clear()
assert view.schema() == {"a": int, "b": int, "computed": float}
assert view2.schema() == {"a": int, "b": int, "computed2": float}
assert view.to_columns() == {}
assert view2.to_columns() == {}
def test_view_expression_with_row_pivots_clear(self):
table = Table({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]})
view = table.view(row_pivots=["computed"],
expressions=[
'// computed \n "a" + "b"',
])
assert view.to_columns() == {
"__ROW_PATH__": [[], [6], [8], [10], [12]],
"a": [10, 1, 2, 3, 4],
"b": [26, 5, 6, 7, 8],
"computed": [36.0, 6.0, 8.0, 10.0, 12.0],
}
table.clear()
assert view.to_columns() == {
"__ROW_PATH__": [[]],
"a": [None],
"b": [None],
"computed": [None],
}
class Demo:
def __init__(self, tb_url: str, stream_key: str, symbol: str, record_type: str, time_widget: widgets.Text, booksize=20):
self.tb_url = tb_url
self.stream_key = stream_key
self.symbol = symbol
self.record_type = record_type
self.time_widget = time_widget
self.booksize = booksize
self.schema = {
'key': str,
'symbol': str,
'side': str,
'size': float,
'price': float,
'numberOfOrders': int
}
self.table = Table(self.schema, limit=booksize * 3, index='key')
self.book = Book(symbol)
self.last_updated = 0
self.stop_reading = False
self.init_book()
def process_entry_update(self, entry: InstrumentMessage) -> None:
if entry.action == 'DELETE':
self.book.remove(entry.side, entry.price)
t = time.time()
if t - self.last_updated >= 0.5:
self.last_updated = t
self.table.update(self.book.get_bids(size=self.booksize))
self.table.update(self.book.get_asks(size=self.booksize))
elif entry.action == 'UPDATE':
e = to_dict(self.symbol, entry)
self.book.update(e)
t = time.time()
if t - self.last_updated >= 0.5:
self.last_updated = t
self.table.update(self.book.get_bids(size=self.booksize))
self.table.update(self.book.get_asks(size=self.booksize))
else:
raise Exception(f'Unknown action type: {entry.action}')
def process_entry_new(self, entry: InstrumentMessage) -> None:
e = to_dict(self.symbol, entry)
self.book.update(e)
t = time.time()
if t - self.last_updated >= 0.5:
self.last_updated = t
self.table.update(self.book.get_bids(size=self.booksize))
self.table.update(self.book.get_asks(size=self.booksize))
def process_snapshot(self, entries) -> None:
self.book.clear()
self.book.update(*map(lambda e: to_dict(self.symbol, e), entries))
t = time.time()
if t - self.last_updated >= 0.5:
self.last_updated = t
self.table.update(self.book.get_bids(size=self.booksize))
self.table.update(self.book.get_asks(size=self.booksize))
def init_book(self):
db = tbapi.TickDb_createFromUrl(self.tb_url)
try:
db.open(True)
stream = db.getStream(self.stream_key)
options = tbapi.SelectionOptions()
try:
cursor = db.select(current_milli_time() - 10000, [stream], options,
[self.record_type],
[self.symbol])
while cursor.next():
msg = cursor.getMessage()
if msg.packageType == 'PERIODICAL_SNAPSHOT':
self.process_snapshot(msg.entries)
break
finally:
cursor.close()
finally:
db.close()
async def read_cursor(self):
db = tbapi.TickDb_createFromUrl(self.tb_url)
try:
db.open(True)
stream = db.getStream(self.stream_key)
options = tbapi.SelectionOptions()
options.live = True
try:
cursor = db.select(current_milli_time(), [stream], options,
[self.record_type],
[self.symbol])
initialized = False
while cursor.next() and not self.stop_reading and not initialized:
msg = cursor.getMessage()
if msg.packageType == 'PERIODICAL_SNAPSHOT' or msg.packageType == 'VENDOR_SNAPSHOT':
logging.info('received snapshot')
self.process_snapshot(msg.entries)
initialized = True
self.time_widget.value = str(datetime.fromtimestamp(msg.timestamp / 10 ** 9))
while cursor.next() and not self.stop_reading:
msg = cursor.getMessage()
if msg.packageType == 'INCREMENTAL_UPDATE':
for entry in msg.entries:
if entry.typeName.endswith('L2EntryUpdate'):
self.process_entry_update(entry)
elif entry.typeName.endswith('L2EntryNew'):
self.process_entry_new(entry)
elif msg.packageType == 'PERIODICAL_SNAPSHOT' or msg.packageType == 'VENDOR_SNAPSHOT':
self.process_snapshot(msg.entries)
self.time_widget.value = str(datetime.fromtimestamp(msg.timestamp / 10 ** 9))
finally:
cursor.close()
finally:
db.close()
def update_table(self):
logging.info('Started streaming!')
loop = asyncio.new_event_loop()
task = loop.create_task(self.read_cursor())
loop.call_later(60, task.cancel)
try:
loop.run_until_complete(task)
except asyncio.CancelledError:
logging.info("Stopped streaming!")
pass
def start(self):
self.stop_reading = False
self.thread = threading.Thread(target=self.update_table)
self.thread.start()
def stop(self):
self.stop_reading = True
self.thread.join()
def clear(self):
self.table.clear()
self.book.clear()
