def test_manager_to_dict_with_options(self): sentinel = False def handle_to_dict(msg): nonlocal sentinel sentinel = True assert msg["data"] == [{"a": 1, "b": "a"}] message = { "id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view" } manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) manager.process(message, self.post) to_dict_message = { "id": 2, "name": "view1", "cmd": "view_method", "method": "to_dict", "args": [{ "start_row": 0, "end_row": 1 }] } manager.process(to_dict_message, handle_to_dict) assert sentinel is True
def test_async_multiple_managers_queue_process_multiple_ports(self): tbl = Table({"a": int, "b": float, "c": str}) tbl2 = Table({"a": int, "b": float, "c": str}) port_ids = [0] port_data = [{"a": 0, "b": 0, "c": "0"}] for i in range(10): port_id = tbl.make_port() port_id2 = tbl2.make_port() assert port_id == port_id2 port_ids.append(port_id) port_data.append({ "a": port_id, "b": port_id * 1.5, "c": str(port_id) }) manager = PerspectiveManager() manager2 = PerspectiveManager() manager.host_table("tbl", tbl) manager2.host_table("tbl2", tbl2) manager.set_loop_callback(TestAsync.loop.add_callback) manager2.set_loop_callback(TestAsync.loop.add_callback) @syncify def _task(): random.shuffle(port_ids) for port_id in port_ids: idx = port_id if port_id < len(port_ids) else len(port_ids) - 1 tbl.update([port_data[idx]], port_id=port_id) tbl2.update([port_data[idx]], port_id=port_id) return (tbl.size(), tbl2.size()) assert _task() == (11, 11)
def test_manager_to_dict(self): sentinel = False def handle_to_dict(msg): nonlocal sentinel sentinel = True assert msg["data"] == data message = { "id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view" } manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) manager.process(message, self.post) to_dict_message = { "id": 2, "name": "view1", "cmd": "view_method", "method": "to_dict" } manager.process(to_dict_message, handle_to_dict) assert sentinel is True
def test_manager_create_indexed_table_and_remove(self): message = { "id": 1, "name": "table1", "cmd": "table", "args": [data], "options": { "index": "a" } } manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) manager._process(message, self.post) assert manager._tables["table1"].schema() == {"a": int, "b": str} assert manager._tables["table1"]._index == "a" remove_message = { "id": 2, "name": "table1", "cmd": "table_method", "method": "remove", "args": [[1, 2]] } manager._process(remove_message, self.post) assert manager._tables["table1"].view().to_dict() == { "a": [3], "b": ["c"] }
def test_manager_set_queue_process_multiple(self, sentinel): # manager2's queue process should not affect manager1, # provided they manage different tables s = sentinel(0) s2 = sentinel(0) manager = PerspectiveManager() manager2 = PerspectiveManager() table = Table({"a": [1, 2, 3]}) table2 = Table({"a": [1, 2, 3]}) manager.host_table("tbl", table) manager2.host_table("tbl2", table2) def fake_queue_process(f, *args, **kwargs): s2.set(s2.get() + 1) f(*args, **kwargs) manager2.set_loop_callback(fake_queue_process) table.update({"a": [4, 5, 6]}) assert table.view().to_dict() == { "a": [1, 2, 3, 4, 5, 6] } table2.update({"a": [7, 8, 9]}) table.update({"a": [7, 8, 9]}) assert table.view().to_dict() == { "a": [1, 2, 3, 4, 5, 6, 7, 8, 9] } assert table2.view().to_dict() == { "a": [1, 2, 3, 7, 8, 9] } assert s.get() == 0 assert s2.get() == 2
def test_manager_clear_view_nonseq(self): messages = [{ "id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view" }, { "id": 2, "table_name": "table1", "view_name": "view2", "cmd": "view" }, { "id": 3, "table_name": "table1", "view_name": "view3", "cmd": "view" }] manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) for i, message in enumerate(messages, 1): manager._process(message, self.post, client_id=i) manager.clear_views(1) manager.clear_views(3) assert "view1" not in manager._views assert "view3" not in manager._views assert "view2" in manager._views
def test_manager_to_dict(self, sentinel): s = sentinel(False) def handle_to_dict(msg): s.set(True) message = json.loads(msg) assert message["data"] == data message = { "id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view" } manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) manager._process(message, self.post) to_dict_message = { "id": 2, "name": "view1", "cmd": "view_method", "method": "to_dict" } manager._process(to_dict_message, handle_to_dict) assert s.get() is True
def test_manager_to_dict_with_nan(self, util, sentinel): data = util.make_arrow(["a"], [[1.5, np.nan, 2.5, np.nan]], types=[pa.float64()]) s = sentinel(False) def handle_to_dict(msg): s.set(True) message = json.loads(msg) assert message == { "id": 2, "error": "JSON serialization error: Cannot serialize `NaN`, `Infinity` or `-Infinity` to JSON." } message = { "id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view" } manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) manager._process(message, self.post) to_dict_message = { "id": 2, "name": "view1", "cmd": "view_method", "method": "to_dict" } manager._process(to_dict_message, handle_to_dict) assert s.get() is True
def test_manager_create_indexed_table_and_update(self): message = { "id": 1, "name": "table1", "cmd": "table", "args": [data], "options": { "index": "a" } } manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) manager._process(message, self.post) assert manager._tables["table1"].schema() == {"a": int, "b": str} assert manager._tables["table1"].get_index() == "a" update_message = { "id": 2, "name": "table1", "cmd": "table_method", "method": "update", "args": [{ "a": [1, 2, 3], "b": ["str1", "str2", "str3"] }] } manager._process(update_message, self.post) assert manager._tables["table1"].view().to_dict() == { "a": [1, 2, 3], "b": ["str1", "str2", "str3"] }
def test_manager_view_expression_schema(self): post_callback = partial(self.validate_post, expected={ "id": 2, "data": { "abc": "float" } }) make_view_message = { "id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view", "config": { "expressions": ['// abc \n "a" + "a"'] } } message = { "id": 2, "name": "view1", "cmd": "view_method", "method": "expression_schema", "args": [False] } manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) manager._process(make_view_message, self.post) manager._process(message, post_callback)
def test_manager_table_validate_expressions(self): post_callback = partial(self.validate_post, expected={ "id": 1, "data": { "expression_schema": { "abc": "float" }, "errors": {}, "expression_alias": { "abc": '// abc \n "a" + "a"' } } }) message = { "id": 1, "name": "table1", "cmd": "table_method", "method": "validate_expressions", "args": [['// abc \n "a" + "a"']] } manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) manager._process(message, post_callback)
def test_manager_view_schema(self): post_callback = partial(self.validate_post, expected={ "id": 2, "data": { "a": "integer", "b": "integer" } }) make_view_message = { "id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view", "config": { "group_by": ["a"] } } message = { "id": 2, "name": "view1", "cmd": "view_method", "method": "schema", "args": [False] } manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) manager._process(make_view_message, self.post) manager._process(message, post_callback)
def test_manager_to_dict_with_options(self, sentinel): s = sentinel(False) def handle_to_dict(msg): s.set(True) message = json.loads(msg) assert message["data"] == {"a": [1], "b": ["a"]} message = { "id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view" } manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) manager._process(message, self.post) to_dict_message = { "id": 2, "name": "view1", "cmd": "view_method", "method": "to_dict", "args": [{ "start_row": 0, "end_row": 1 }] } manager._process(to_dict_message, handle_to_dict) assert s.get() is True
def make_app(): # Create an instance of `PerspectiveManager` and a table. MANAGER = PerspectiveManager() TABLE = Table( { "name": str, "client": str, "open": float, "high": float, "low": float, "close": float, "lastUpdate": datetime, }, limit=2500) # Track the table with the name "data_source_one", which will be used in # the front-end to access the Table. MANAGER.host_table("data_source_one", TABLE) # update with new data every 50ms def updater(): TABLE.update(data_source()) callback = tornado.ioloop.PeriodicCallback(callback=updater, callback_time=50) callback.start() return tornado.web.Application([ (r"/", MainHandler), # create a websocket endpoint that the client Javascript can access (r"/websocket", PerspectiveTornadoHandler, { "manager": MANAGER, "check_origin": True }) ])
def make_app(): # Create an instance of `PerspectiveManager` and a table. MANAGER = PerspectiveManager(lock=True) TABLE = Table({ "name": str, "client": str, "open": float, "high": float, "low": float, "close": float, "lastUpdate": datetime, }) TABLE.update(data_source(100)) # Track the table with the name "data_source_one", which will be used in # the front-end to access the Table. MANAGER.host_table("data_source_one", TABLE) return tornado.web.Application([ (r"/", MainHandler), # create a websocket endpoint that the client Javascript can access (r"/websocket", PerspectiveTornadoHandler, { "manager": MANAGER, "check_origin": True }) ])
def test_manager_to_dict_unix_timestamps_from_datetime(self, sentinel): """The conversion from `datetime` to a Unix timestamp should not alter the timestamp in any way if both are in local time.""" s = sentinel(False) timestamp_data = { "a": [1580515140000] } schema = { "a": datetime } def handle_to_dict(msg): s.set(True) message = json.loads(msg) assert message["data"] == timestamp_data # convert back ts = datetime.fromtimestamp(message["data"]["a"][0] / 1000) assert ts == datetime(2020, 1, 31, 23, 59) message = {"id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view"} manager = PerspectiveManager() table = Table(schema) table.update(timestamp_data) manager.host_table("table1", table) manager._process(message, self.post) to_dict_message = {"id": 2, "name": "view1", "cmd": "view_method", "method": "to_dict"} manager._process(to_dict_message, handle_to_dict) assert s.get() is True
def test_manager_create_view_zero(self): message = {"id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view"} manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) manager._process(message, self.post) assert manager._views["view1"].num_rows() == 3
def test_manager_table_computed_schema(self): post_callback = partial(self.validate_post, expected={ "id": 1, "data": { "abc": "float" } }) message = { "id": 1, "name": "table1", "cmd": "table_method", "method": "computed_schema", "args": [ [ { "column": "abc", "computed_function_name": "+", "inputs": ["a", "a"] } ] ] } manager = PerspectiveManager() table = Table(data) view = table.view() manager.host_table("table1", table) manager._process(message, post_callback)
def test_async_multiple_managers_queue_process(self): tbl = Table({ "a": int, "b": float, "c": str }) tbl2 = Table({ "a": int, "b": float, "c": str }) manager = PerspectiveManager() manager2 = PerspectiveManager() manager.host_table("tbl", tbl) manager2.host_table("tbl2", tbl2) manager._set_queue_process(TestAsync.wrapped_queue_process) manager2._set_queue_process(TestAsync.wrapped_queue_process) for i in range(5): tbl.update([data[i]]) tbl2.update([data[i]]) assert SENTINEL.get() != 0 # flush `TO_PROCESS` assert tbl.view().to_records() == data[:5] for i in range(5): tbl2.update([data[i]])
def test_async_multiple_managers_mixed_queue_process(self): # mutate when synchronously calling queue_process for each update SENTINEL_2 = AsyncSentinel(0) def sync_queue_process(table_id, state_manager): SENTINEL_2.set(SENTINEL_2.get() - 1) state_manager.call_process(table_id) tbl = Table({"a": int, "b": float, "c": str}) tbl2 = Table({"a": int, "b": float, "c": str}) manager = PerspectiveManager() manager2 = PerspectiveManager() manager.host_table("tbl", tbl) manager2.host_table("tbl2", tbl2) # manager uses tornado, manager2 is synchronous manager._set_queue_process(TestAsync.wrapped_queue_process) manager2._set_queue_process(sync_queue_process) tbl_id = tbl._table.get_id() tbl2_id = tbl2._table.get_id() for i in range(5): tbl.update([data[i]]) tbl2.update([data[i]]) assert SENTINEL.get() != 0 assert SENTINEL_2.get() == -5 assert tbl2_id not in _PerspectiveStateManager.TO_PROCESS # flush `TO_PROCESS` assert tbl.view().to_records() == data[:5] assert tbl_id not in _PerspectiveStateManager.TO_PROCESS
def make_app(): with open(file_path, mode='rb') as file: # Create an instance of `PerspectiveManager` and a table. MANAGER = PerspectiveManager() TABLE = Table(file.read(), index="Row ID") # Track the table with the name "data_source_one", which will be used # in the front-end to access the Table. MANAGER.host_table("data_source_one", TABLE) MANAGER.host_view("view_one", TABLE.view()) return tornado.web.Application( [ # create a websocket endpoint that the client Javascript can access (r"/websocket", PerspectiveTornadoHandler, { "manager": MANAGER, "check_origin": True }), (r"/node_modules/(.*)", tornado.web.StaticFileHandler, { "path": "../../node_modules/@finos/" }), (r"/(.*)", tornado.web.StaticFileHandler, { "path": "./", "default_filename": "index.html" }) ], websocket_ping_interval=15)
def test_manager_create_view_and_update_table(self): message = {"id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view"} manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) manager._process(message, self.post) table.update([{"a": 4, "b": "d"}]) assert manager._views["view1"].num_rows() == 4
def test_async_multiple_managers_mixed_queue_process_multiple_ports(self): sentinel = {"async": 0, "sync": 0} def _counter(key, f, *args, **kwargs): sentinel[key] += 1 return f(*args, **kwargs) sync_process = partial(_counter, "sync") async_process = partial(TestAsync.loop.add_timeout, 1, _counter, "async") tbl = Table({"a": int, "b": float, "c": str}) tbl2 = Table({"a": int, "b": float, "c": str}) port_ids = [0] port_data = [{"a": 0, "b": 0, "c": "0"}] for i in range(10): port_id = tbl.make_port() port_id2 = tbl2.make_port() assert port_id == port_id2 port_ids.append(port_id) port_data.append({ "a": port_id, "b": port_id * 1.5, "c": str(port_id) }) manager = PerspectiveManager() manager2 = PerspectiveManager() manager.host_table("tbl", tbl) manager2.host_table("tbl2", tbl2) # manager uses tornado, manager2 is synchronous manager.set_loop_callback(async_process) manager2.set_loop_callback(sync_process) random.shuffle(port_ids) @syncify def _task(): for port_id in port_ids: idx = port_id if port_id < len(port_ids) else len(port_ids) - 1 tbl.update([port_data[idx]], port_id=port_id) _task() for port_id in port_ids: idx = port_id if port_id < len(port_ids) else len(port_ids) - 1 tbl2.update([port_data[idx]], port_id=port_id) @syncify def _get_size(): size = tbl.size() tbl.delete() return size assert _get_size() == 11 assert tbl2.size() == 11 assert sentinel["async"] == 2 assert sentinel["sync"] == 12 tbl2.delete()
def test_manager_host_table(self): message = {"id": 1, "name": "table1", "cmd": "table_method", "method": "schema", "args": []} manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) manager._process(message, self.post) assert manager._tables["table1"].schema() == { "a": int, "b": str }
def test_locked_manager_create_view(self): message = {"id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view"} manager = PerspectiveManager(lock=True) table = Table(data) manager.host_table("table1", table) manager._process(message, self.post) assert manager._get_view("view1").schema() == { "a": int, "b": str }
def test_manager_create_view_one(self): message = {"id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view", "config": {"row_pivots": ["a"]}} manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) manager._process(message, self.post) assert manager._views["view1"].to_dict() == { "__ROW_PATH__": [[], [1], [2], [3]], "a": [6, 1, 2, 3], "b": [3, 1, 1, 1] }
def test_manager_create_indexed_table(self): message = {"id": 1, "name": "table1", "cmd": "table", "args": [data], "options": {"index": "a"}} manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) manager._process(message, self.post) assert manager._tables["table1"].schema() == { "a": int, "b": str } assert manager._tables["table1"].get_index() == "a"
def test_locked_manager_create_view(self): message = {"id": 1, "name": "view1", "cmd": "view_method", "method": "schema", "args": []} manager = PerspectiveManager(lock=True) table = Table(data) view = table.view() manager.host_table("table1", table) manager.host_view("view1", view) manager._process(message, self.post) assert manager.get_view("view1").schema() == { "a": int, "b": str }
def test_manager_clear_view_no_client_id(self): messages = [ {"id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view"}, {"id": 2, "table_name": "table1", "view_name": "view2", "cmd": "view"}, {"id": 3, "table_name": "table1", "view_name": "view3", "cmd": "view"} ] manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) for message in messages: manager._process(message, self.post) with raises(PerspectiveError): manager.clear_views(None)
def test_manager_clear_view(self): messages = [ {"id": 1, "table_name": "table1", "view_name": "view1", "cmd": "view"}, {"id": 2, "table_name": "table1", "view_name": "view2", "cmd": "view"}, {"id": 3, "table_name": "table1", "view_name": "view3", "cmd": "view"} ] manager = PerspectiveManager() table = Table(data) manager.host_table("table1", table) for message in messages: manager._process(message, self.post, client_id=1) manager.clear_views(1) assert manager._views == {}