def test_manager_to_dict_with_options(self):
sentinel = False
def handle_to_dict(msg):
nonlocal sentinel
sentinel = True
assert msg["data"] == [{"a": 1, "b": "a"}]
message = {
"id": 1,
"table_name": "table1",
"view_name": "view1",
"cmd": "view"
}
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
manager.process(message, self.post)
to_dict_message = {
"id": 2,
"name": "view1",
"cmd": "view_method",
"method": "to_dict",
"args": [{
"start_row": 0,
"end_row": 1
}]
}
manager.process(to_dict_message, handle_to_dict)
assert sentinel is True
def test_async_multiple_managers_queue_process_multiple_ports(self):
tbl = Table({"a": int, "b": float, "c": str})
tbl2 = Table({"a": int, "b": float, "c": str})
port_ids = [0]
port_data = [{"a": 0, "b": 0, "c": "0"}]
for i in range(10):
port_id = tbl.make_port()
port_id2 = tbl2.make_port()
assert port_id == port_id2
port_ids.append(port_id)
port_data.append({
"a": port_id,
"b": port_id * 1.5,
"c": str(port_id)
})
manager = PerspectiveManager()
manager2 = PerspectiveManager()
manager.host_table("tbl", tbl)
manager2.host_table("tbl2", tbl2)
manager.set_loop_callback(TestAsync.loop.add_callback)
manager2.set_loop_callback(TestAsync.loop.add_callback)
@syncify
def _task():
random.shuffle(port_ids)
for port_id in port_ids:
idx = port_id if port_id < len(port_ids) else len(port_ids) - 1
tbl.update([port_data[idx]], port_id=port_id)
tbl2.update([port_data[idx]], port_id=port_id)
return (tbl.size(), tbl2.size())
assert _task() == (11, 11)
def test_manager_to_dict(self):
sentinel = False
def handle_to_dict(msg):
nonlocal sentinel
sentinel = True
assert msg["data"] == data
message = {
"id": 1,
"table_name": "table1",
"view_name": "view1",
"cmd": "view"
}
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
manager.process(message, self.post)
to_dict_message = {
"id": 2,
"name": "view1",
"cmd": "view_method",
"method": "to_dict"
}
manager.process(to_dict_message, handle_to_dict)
assert sentinel is True
def test_manager_create_indexed_table_and_remove(self):
message = {
"id": 1,
"name": "table1",
"cmd": "table",
"args": [data],
"options": {
"index": "a"
}
}
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
manager._process(message, self.post)
assert manager._tables["table1"].schema() == {"a": int, "b": str}
assert manager._tables["table1"]._index == "a"
remove_message = {
"id": 2,
"name": "table1",
"cmd": "table_method",
"method": "remove",
"args": [[1, 2]]
}
manager._process(remove_message, self.post)
assert manager._tables["table1"].view().to_dict() == {
"a": [3],
"b": ["c"]
}
def test_manager_set_queue_process_multiple(self, sentinel):
# manager2's queue process should not affect manager1,
# provided they manage different tables
s = sentinel(0)
s2 = sentinel(0)
manager = PerspectiveManager()
manager2 = PerspectiveManager()
table = Table({"a": [1, 2, 3]})
table2 = Table({"a": [1, 2, 3]})
manager.host_table("tbl", table)
manager2.host_table("tbl2", table2)
def fake_queue_process(f, *args, **kwargs):
s2.set(s2.get() + 1)
f(*args, **kwargs)
manager2.set_loop_callback(fake_queue_process)
table.update({"a": [4, 5, 6]})
assert table.view().to_dict() == {
"a": [1, 2, 3, 4, 5, 6]
}
table2.update({"a": [7, 8, 9]})
table.update({"a": [7, 8, 9]})
assert table.view().to_dict() == {
"a": [1, 2, 3, 4, 5, 6, 7, 8, 9]
}
assert table2.view().to_dict() == {
"a": [1, 2, 3, 7, 8, 9]
}
assert s.get() == 0
assert s2.get() == 2
def test_manager_clear_view_nonseq(self):
messages = [{
"id": 1,
"table_name": "table1",
"view_name": "view1",
"cmd": "view"
}, {
"id": 2,
"table_name": "table1",
"view_name": "view2",
"cmd": "view"
}, {
"id": 3,
"table_name": "table1",
"view_name": "view3",
"cmd": "view"
}]
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
for i, message in enumerate(messages, 1):
manager._process(message, self.post, client_id=i)
manager.clear_views(1)
manager.clear_views(3)
assert "view1" not in manager._views
assert "view3" not in manager._views
assert "view2" in manager._views
def test_manager_to_dict(self, sentinel):
s = sentinel(False)
def handle_to_dict(msg):
s.set(True)
message = json.loads(msg)
assert message["data"] == data
message = {
"id": 1,
"table_name": "table1",
"view_name": "view1",
"cmd": "view"
}
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
manager._process(message, self.post)
to_dict_message = {
"id": 2,
"name": "view1",
"cmd": "view_method",
"method": "to_dict"
}
manager._process(to_dict_message, handle_to_dict)
assert s.get() is True
def test_manager_to_dict_with_nan(self, util, sentinel):
data = util.make_arrow(["a"], [[1.5, np.nan, 2.5, np.nan]],
types=[pa.float64()])
s = sentinel(False)
def handle_to_dict(msg):
s.set(True)
message = json.loads(msg)
assert message == {
"id":
2,
"error":
"JSON serialization error: Cannot serialize `NaN`, `Infinity` or `-Infinity` to JSON."
}
message = {
"id": 1,
"table_name": "table1",
"view_name": "view1",
"cmd": "view"
}
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
manager._process(message, self.post)
to_dict_message = {
"id": 2,
"name": "view1",
"cmd": "view_method",
"method": "to_dict"
}
manager._process(to_dict_message, handle_to_dict)
assert s.get() is True
def test_manager_create_indexed_table_and_update(self):
message = {
"id": 1,
"name": "table1",
"cmd": "table",
"args": [data],
"options": {
"index": "a"
}
}
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
manager._process(message, self.post)
assert manager._tables["table1"].schema() == {"a": int, "b": str}
assert manager._tables["table1"].get_index() == "a"
update_message = {
"id": 2,
"name": "table1",
"cmd": "table_method",
"method": "update",
"args": [{
"a": [1, 2, 3],
"b": ["str1", "str2", "str3"]
}]
}
manager._process(update_message, self.post)
assert manager._tables["table1"].view().to_dict() == {
"a": [1, 2, 3],
"b": ["str1", "str2", "str3"]
}
def test_manager_view_expression_schema(self):
post_callback = partial(self.validate_post,
expected={
"id": 2,
"data": {
"abc": "float"
}
})
make_view_message = {
"id": 1,
"table_name": "table1",
"view_name": "view1",
"cmd": "view",
"config": {
"expressions": ['// abc \n "a" + "a"']
}
}
message = {
"id": 2,
"name": "view1",
"cmd": "view_method",
"method": "expression_schema",
"args": [False]
}
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
manager._process(make_view_message, self.post)
manager._process(message, post_callback)
def test_manager_table_validate_expressions(self):
post_callback = partial(self.validate_post,
expected={
"id": 1,
"data": {
"expression_schema": {
"abc": "float"
},
"errors": {},
"expression_alias": {
"abc": '// abc \n "a" + "a"'
}
}
})
message = {
"id": 1,
"name": "table1",
"cmd": "table_method",
"method": "validate_expressions",
"args": [['// abc \n "a" + "a"']]
}
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
manager._process(message, post_callback)
def test_manager_view_schema(self):
post_callback = partial(self.validate_post,
expected={
"id": 2,
"data": {
"a": "integer",
"b": "integer"
}
})
make_view_message = {
"id": 1,
"table_name": "table1",
"view_name": "view1",
"cmd": "view",
"config": {
"group_by": ["a"]
}
}
message = {
"id": 2,
"name": "view1",
"cmd": "view_method",
"method": "schema",
"args": [False]
}
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
manager._process(make_view_message, self.post)
manager._process(message, post_callback)
def test_manager_to_dict_with_options(self, sentinel):
s = sentinel(False)
def handle_to_dict(msg):
s.set(True)
message = json.loads(msg)
assert message["data"] == {"a": [1], "b": ["a"]}
message = {
"id": 1,
"table_name": "table1",
"view_name": "view1",
"cmd": "view"
}
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
manager._process(message, self.post)
to_dict_message = {
"id": 2,
"name": "view1",
"cmd": "view_method",
"method": "to_dict",
"args": [{
"start_row": 0,
"end_row": 1
}]
}
manager._process(to_dict_message, handle_to_dict)
assert s.get() is True
def make_app():
# Create an instance of `PerspectiveManager` and a table.
MANAGER = PerspectiveManager()
TABLE = Table(
{
"name": str,
"client": str,
"open": float,
"high": float,
"low": float,
"close": float,
"lastUpdate": datetime,
},
limit=2500)
# Track the table with the name "data_source_one", which will be used in
# the front-end to access the Table.
MANAGER.host_table("data_source_one", TABLE)
# update with new data every 50ms
def updater():
TABLE.update(data_source())
callback = tornado.ioloop.PeriodicCallback(callback=updater,
callback_time=50)
callback.start()
return tornado.web.Application([
(r"/", MainHandler),
# create a websocket endpoint that the client Javascript can access
(r"/websocket", PerspectiveTornadoHandler, {
"manager": MANAGER,
"check_origin": True
})
])
def make_app():
# Create an instance of `PerspectiveManager` and a table.
MANAGER = PerspectiveManager(lock=True)
TABLE = Table({
"name": str,
"client": str,
"open": float,
"high": float,
"low": float,
"close": float,
"lastUpdate": datetime,
})
TABLE.update(data_source(100))
# Track the table with the name "data_source_one", which will be used in
# the front-end to access the Table.
MANAGER.host_table("data_source_one", TABLE)
return tornado.web.Application([
(r"/", MainHandler),
# create a websocket endpoint that the client Javascript can access
(r"/websocket", PerspectiveTornadoHandler, {
"manager": MANAGER,
"check_origin": True
})
])
def test_manager_to_dict_unix_timestamps_from_datetime(self, sentinel):
"""The conversion from `datetime` to a Unix timestamp should not
alter the timestamp in any way if both are in local time."""
s = sentinel(False)
timestamp_data = {
"a": [1580515140000]
}
schema = {
"a": datetime
}
def handle_to_dict(msg):
s.set(True)
message = json.loads(msg)
assert message["data"] == timestamp_data
# convert back
ts = datetime.fromtimestamp(message["data"]["a"][0] / 1000)
assert ts == datetime(2020, 1, 31, 23, 59)
message = {"id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view"}
manager = PerspectiveManager()
table = Table(schema)
table.update(timestamp_data)
manager.host_table("table1", table)
manager._process(message, self.post)
to_dict_message = {"id": 2, "name": "view1", "cmd": "view_method", "method": "to_dict"}
manager._process(to_dict_message, handle_to_dict)
assert s.get() is True
def test_manager_create_view_zero(self):
message = {"id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view"}
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
manager._process(message, self.post)
assert manager._views["view1"].num_rows() == 3
def test_manager_table_computed_schema(self):
post_callback = partial(self.validate_post, expected={
"id": 1,
"data": {
"abc": "float"
}
})
message = {
"id": 1,
"name": "table1",
"cmd": "table_method",
"method": "computed_schema",
"args": [
[
{
"column": "abc",
"computed_function_name": "+",
"inputs": ["a", "a"]
}
]
]
}
manager = PerspectiveManager()
table = Table(data)
view = table.view()
manager.host_table("table1", table)
manager._process(message, post_callback)
def test_async_multiple_managers_queue_process(self):
tbl = Table({
"a": int,
"b": float,
"c": str
})
tbl2 = Table({
"a": int,
"b": float,
"c": str
})
manager = PerspectiveManager()
manager2 = PerspectiveManager()
manager.host_table("tbl", tbl)
manager2.host_table("tbl2", tbl2)
manager._set_queue_process(TestAsync.wrapped_queue_process)
manager2._set_queue_process(TestAsync.wrapped_queue_process)
for i in range(5):
tbl.update([data[i]])
tbl2.update([data[i]])
assert SENTINEL.get() != 0
# flush `TO_PROCESS`
assert tbl.view().to_records() == data[:5]
for i in range(5):
tbl2.update([data[i]])
def test_async_multiple_managers_mixed_queue_process(self):
# mutate when synchronously calling queue_process for each update
SENTINEL_2 = AsyncSentinel(0)
def sync_queue_process(table_id, state_manager):
SENTINEL_2.set(SENTINEL_2.get() - 1)
state_manager.call_process(table_id)
tbl = Table({"a": int, "b": float, "c": str})
tbl2 = Table({"a": int, "b": float, "c": str})
manager = PerspectiveManager()
manager2 = PerspectiveManager()
manager.host_table("tbl", tbl)
manager2.host_table("tbl2", tbl2)
# manager uses tornado, manager2 is synchronous
manager._set_queue_process(TestAsync.wrapped_queue_process)
manager2._set_queue_process(sync_queue_process)
tbl_id = tbl._table.get_id()
tbl2_id = tbl2._table.get_id()
for i in range(5):
tbl.update([data[i]])
tbl2.update([data[i]])
assert SENTINEL.get() != 0
assert SENTINEL_2.get() == -5
assert tbl2_id not in _PerspectiveStateManager.TO_PROCESS
# flush `TO_PROCESS`
assert tbl.view().to_records() == data[:5]
assert tbl_id not in _PerspectiveStateManager.TO_PROCESS
def make_app():
with open(file_path, mode='rb') as file:
# Create an instance of `PerspectiveManager` and a table.
MANAGER = PerspectiveManager()
TABLE = Table(file.read(), index="Row ID")
# Track the table with the name "data_source_one", which will be used
# in the front-end to access the Table.
MANAGER.host_table("data_source_one", TABLE)
MANAGER.host_view("view_one", TABLE.view())
return tornado.web.Application(
[
# create a websocket endpoint that the client Javascript can access
(r"/websocket", PerspectiveTornadoHandler, {
"manager": MANAGER,
"check_origin": True
}),
(r"/node_modules/(.*)", tornado.web.StaticFileHandler, {
"path": "../../node_modules/@finos/"
}),
(r"/(.*)", tornado.web.StaticFileHandler, {
"path": "./",
"default_filename": "index.html"
})
],
websocket_ping_interval=15)
def test_manager_create_view_and_update_table(self):
message = {"id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view"}
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
manager._process(message, self.post)
table.update([{"a": 4, "b": "d"}])
assert manager._views["view1"].num_rows() == 4
def test_async_multiple_managers_mixed_queue_process_multiple_ports(self):
sentinel = {"async": 0, "sync": 0}
def _counter(key, f, *args, **kwargs):
sentinel[key] += 1
return f(*args, **kwargs)
sync_process = partial(_counter, "sync")
async_process = partial(TestAsync.loop.add_timeout, 1, _counter,
"async")
tbl = Table({"a": int, "b": float, "c": str})
tbl2 = Table({"a": int, "b": float, "c": str})
port_ids = [0]
port_data = [{"a": 0, "b": 0, "c": "0"}]
for i in range(10):
port_id = tbl.make_port()
port_id2 = tbl2.make_port()
assert port_id == port_id2
port_ids.append(port_id)
port_data.append({
"a": port_id,
"b": port_id * 1.5,
"c": str(port_id)
})
manager = PerspectiveManager()
manager2 = PerspectiveManager()
manager.host_table("tbl", tbl)
manager2.host_table("tbl2", tbl2)
# manager uses tornado, manager2 is synchronous
manager.set_loop_callback(async_process)
manager2.set_loop_callback(sync_process)
random.shuffle(port_ids)
@syncify
def _task():
for port_id in port_ids:
idx = port_id if port_id < len(port_ids) else len(port_ids) - 1
tbl.update([port_data[idx]], port_id=port_id)
_task()
for port_id in port_ids:
idx = port_id if port_id < len(port_ids) else len(port_ids) - 1
tbl2.update([port_data[idx]], port_id=port_id)
@syncify
def _get_size():
size = tbl.size()
tbl.delete()
return size
assert _get_size() == 11
assert tbl2.size() == 11
assert sentinel["async"] == 2
assert sentinel["sync"] == 12
tbl2.delete()
def test_manager_host_table(self):
message = {"id": 1, "name": "table1", "cmd": "table_method", "method": "schema", "args": []}
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
manager._process(message, self.post)
assert manager._tables["table1"].schema() == {
"a": int,
"b": str
}
def test_locked_manager_create_view(self):
message = {"id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view"}
manager = PerspectiveManager(lock=True)
table = Table(data)
manager.host_table("table1", table)
manager._process(message, self.post)
assert manager._get_view("view1").schema() == {
"a": int,
"b": str
}
def test_manager_create_view_one(self):
message = {"id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view", "config": {"row_pivots": ["a"]}}
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
manager._process(message, self.post)
assert manager._views["view1"].to_dict() == {
"__ROW_PATH__": [[], [1], [2], [3]],
"a": [6, 1, 2, 3],
"b": [3, 1, 1, 1]
}
def test_manager_create_indexed_table(self):
message = {"id": 1, "name": "table1", "cmd": "table", "args": [data], "options": {"index": "a"}}
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
manager._process(message, self.post)
assert manager._tables["table1"].schema() == {
"a": int,
"b": str
}
assert manager._tables["table1"].get_index() == "a"
def test_locked_manager_create_view(self):
message = {"id": 1, "name": "view1", "cmd": "view_method", "method": "schema", "args": []}
manager = PerspectiveManager(lock=True)
table = Table(data)
view = table.view()
manager.host_table("table1", table)
manager.host_view("view1", view)
manager._process(message, self.post)
assert manager.get_view("view1").schema() == {
"a": int,
"b": str
}
def test_manager_clear_view_no_client_id(self):
messages = [
{"id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view"},
{"id": 2, "table_name": "table1", "view_name": "view2", "cmd": "view"},
{"id": 3, "table_name": "table1", "view_name": "view3", "cmd": "view"}
]
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
for message in messages:
manager._process(message, self.post)
with raises(PerspectiveError):
manager.clear_views(None)
def test_manager_clear_view(self):
messages = [
{"id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view"},
{"id": 2, "table_name": "table1", "view_name": "view2", "cmd": "view"},
{"id": 3, "table_name": "table1", "view_name": "view3", "cmd": "view"}
]
manager = PerspectiveManager()
table = Table(data)
manager.host_table("table1", table)
for message in messages:
manager._process(message, self.post, client_id=1)
manager.clear_views(1)
assert manager._views == {}
