def test_catch_broken_queue_error(event_loop: AbstractEventLoop) -> None: """test for deleting log queue when read queue is awaiting streaming data from log queue""" logstreamer = ToyLogStreamer([log_streamer.QueueHandler()]) read_streamer = LocalReadStreamer(logstreamer) async def flow(read_streamer: LocalReadStreamer): id = "xxx" key = "key-something" # add pipe between log streamer and application read streamer await read_streamer.add_client(id, key) assert read_streamer._id_to_logqueue.get(id) assert read_streamer._key_to_readqueue.get(key) with pytest.raises(ValueError): # keep awaitting to read streamer get log until timeout await read_streamer.get(key) # successfully raise error and break all running asynchronous process async def post_flow(logstreamer): await asyncio.sleep(0.3) # force delete log queue logstreamer.delete("xxx") await asyncio.sleep(0.3) event_loop.create_task(flow(read_streamer)) event_loop.run_until_complete(post_flow(logstreamer)) event_loop.close()
async def flow(): logstreamer = ToyLogStreamer([log_streamer.QueueHandler()]) read_streamer = LocalReadStreamer(logstreamer) id = "xxx" key = "key-something" # add pipe between log streamer and application read streamer await read_streamer.add_client(id, key) assert read_streamer._id_to_logqueue.get(id), "log queue not found" assert read_streamer._key_to_readqueue.get(key), "read queue not found" assert logstreamer._handlers[0]._queues, "log streamer is empty" assert read_streamer._id_to_logqueue.get(id).task, "task is not running" # log streamer receive log and send it into queue for read streamer await logstreamer.streaming() await asyncio.sleep(0.01) # read streamer is collectting streaming data asynchronously # read streamer get log out = await read_streamer.get(key) assert out == "test\n" # finish streaming await read_streamer.delete(key) with pytest.raises(KeyError): # can't get data anymore await read_streamer.get(key)
def test_log_trace() -> None: logstreamer = ToyLogStreamer([log_streamer.QueueHandler()]) read_streamer = LocalReadStreamer(logstreamer) app = create_app(read_streamer) async def flow(logstreamer: ToyLogStreamer) -> None: # instantiate asyncio.Queue in current event loop # log streamer receive log and send it into queue for read streamer logstreamer.add("xxx") await logstreamer.streaming() await logstreamer.streaming() client = TestClient(app) with client.websocket_connect("/log-trace?id=xxx") as websocket: # running app new event loop in subthread subloop: AbstractEventLoop = websocket._loop # inject coroutine that streaming log data and wait to finish future = asyncio.run_coroutine_threadsafe(flow(logstreamer), subloop) future.result() future.cancel() # successfully log tracked data = websocket.receive_text() assert data == "test\n" websocket.close() # wait closing background process # socket is closed and queues in log streamer and read streamer assert not logstreamer._handlers[0]._queues assert not read_streamer._key_to_readqueue
def test_delete_internal_task(event_loop: AbstractEventLoop) -> None: """test for deleting log queue when read queue is awaiting streaming data from log queue""" logstreamer = ToyLogStreamer([log_streamer.QueueHandler()]) read_streamer = LocalReadStreamer(logstreamer) async def flow(read_streamer: LocalReadStreamer, logstreamer: ToyLogStreamer): id = "xxx" key = "key-something" # add pipe between log streamer and application read streamer await read_streamer.add_client(id, key) assert read_streamer._id_to_logqueue.get(id) assert read_streamer._key_to_readqueue.get(key) # running task for sync streamer log = read_streamer._id_to_logqueue.get(id) assert log assert not log.task.cancelled() await logstreamer.streaming() await logstreamer.streaming() assert not log.task.cancelled() # cancel sync await read_streamer.delete(key) await asyncio.sleep(0.3) assert log.task.cancelled() event_loop.run_until_complete(flow(read_streamer, logstreamer)) event_loop.close()
def main( http: bool = typer.Option(True, "-h", help="connect via http"), queue: Queues = typer.Option("file", "-q", help="select queue"), logger: Loggers = typer.Option("stream", "-l", help="select logger"), streamer: log_streamer.LogStreamers = typer.Option( "local", "-s", help="select log streamer"), frequency: float = typer.Option(3, "--freq", help="worker inspection frequency [sec]"), ) -> None: """Managements Runner for: - Worker: run or wait worker subprocess for the latest job in queue, including logging. - Queue: CRUD for Queue (add job, get jobs, ...) """ dep = CopyDep(None, BASEDIR / "dep") queue_ = QUEUE_CLASSES[queue](path=BASEDIR / "queue", depends=dep) logger_ = LOGGER_CLASSES[logger]() loop = asyncio.get_event_loop() loop.set_debug(False) worker = Worker(logger_, queue_, freq=frequency) if http: log_streamer_handler = log_streamer.QueueHandler() log_streamer_class = log_streamer.LOGSTREAMER_CLASSES[streamer] if log_streamer_class == log_streamer.LocalLogStreamer and isinstance( logger_, StreamingLogger): log_streamer_ = log_streamer.LocalLogStreamer( [ log_streamer.QueueFileHandler(str(BASEDIR / "log")), log_streamer_handler, ], logger_, ) read_streamer = LocalReadStreamer(log_streamer_) app = create_app(read_streamer) server = run_receiver( app, loop, [worker.handle_exit, log_streamer_.handle_exit]) loop.create_task(server.serve()) loop.create_task(log_streamer_.entry_point()) try: loop.run_until_complete(worker._run(loop)) except KeyboardInterrupt: pass
async def test_localread_streamer_failure(): # QueueFileHandler is required logstreamer = ToyLogStreamer([]) with pytest.raises(ValueError): LocalReadStreamer(logstreamer) logstreamer = ToyLogStreamer([log_streamer.QueueHandler()]) read_streamer = LocalReadStreamer(logstreamer) id = "xxx" key = "key-something" # add pipe between log streamer and application read streamer await read_streamer.add_client(id, key) # broken connection, can't get anymore read_streamer._key_to_readqueue.get(key).live = False with pytest.raises(ValueError): await read_streamer.get(key)
def test_queue_handler(event_loop: AbstractEventLoop) -> None: handler = log_streamer.QueueHandler() event_loop.run_until_complete(workflow(handler)) queues = handler._queues assert queues.get("xxx").get_nowait() == "test-x\n" assert queues.get("yyy").get_nowait() == "test-y\n" assert not queues.get("zzz") async def delete(handler): # raise await handler.delete("aaa") # pop successfully await handler.delete("xxx") event_loop.run_until_complete(delete(handler)) assert not queues.get("xxx")
async def test_preload(): with tempfile.TemporaryDirectory() as tempdir: logstreamer = ToyLogStreamer( [log_streamer.QueueHandler(), log_streamer.QueueFileHandler(tempdir)] ) read_streamer = LocalReadStreamer(logstreamer) id = "xxx" key = "key-something-x" logstreamer.id = id logstreamer.msg = "test\n" await logstreamer.streaming() await asyncio.sleep(0.05) await read_streamer.add_client(id, key) data = await read_streamer.get(key) assert data == "test\n\n", "additional newline is inserted" data = await read_streamer.get(key) assert data == "-------------- loading -------------\n" id = "yyy" key = "key-something-y" logstreamer.id = id logstreamer.msg = "test" await logstreamer.streaming() await logstreamer.streaming() await asyncio.sleep(0.05) await read_streamer.add_client(id, key) data = await read_streamer.get(key) assert data == "testtest\n", "additional newline at only eof" data = await read_streamer.get(key) assert data == "-------------- loading -------------\n" id = "zzz" key = "key-something-z" logstreamer.id = id logstreamer.msg = "test\r" await logstreamer.streaming() await logstreamer.streaming() await asyncio.sleep(0.05) await read_streamer.add_client(id, key) data = await read_streamer.get(key) assert data == "test\rtest\r\n", "\\r is not eliminated" data = await read_streamer.get(key) assert data == "-------------- loading -------------\n"