def test_tcp_server_unix(unix_socket_tcp): class TestService(TCPServer): DATA = [] async def handle_client(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): self.DATA.append(await reader.readline()) writer.close() service = TestService(sock=unix_socket_tcp) @aiomisc.threaded def writer(): with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: sock.connect(unix_socket_tcp.getsockname()) sock.send(b"hello server\n") with aiomisc.entrypoint(service) as loop: loop.run_until_complete(asyncio.wait_for(writer(), timeout=10), ) assert TestService.DATA assert TestService.DATA == [b"hello server\n"]
def test_service_events(): class Initialization(aiomisc.Service): async def start(self): context = aiomisc.get_context() await asyncio.sleep(0.1) context["test"] = True class Awaiter(aiomisc.Service): result = None async def start(self): Awaiter.result = await self.context["test"] services = ( Awaiter(), Initialization(), ) with aiomisc.entrypoint(*services): pass assert Awaiter.result
def test_tcp_server(aiomisc_unused_port): class TestService(TCPServer): DATA = [] async def handle_client(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): self.DATA.append(await reader.readline()) writer.close() service = TestService("127.0.0.1", aiomisc_unused_port) @aiomisc.threaded def writer(): port = aiomisc_unused_port with socket.create_connection(("127.0.0.1", port)) as sock: sock.send(b"hello server\n") with aiomisc.entrypoint(service) as loop: loop.run_until_complete(asyncio.wait_for(writer(), timeout=10), ) assert TestService.DATA assert TestService.DATA == [b"hello server\n"]
async def test_entrypoint_with_with_async(): class MyService(aiomisc.service.Service): ctx = 0 async def start(self): self.__class__.ctx = 1 async def stop(self, exc: Exception = None): self.__class__.ctx = 2 service = MyService() assert service.ctx == 0 async with aiomisc.entrypoint(service) as ep: assert service.ctx == 1 with pytest.raises(asyncio.TimeoutError): await asyncio.wait_for(ep.closing(), timeout=0.3) assert service.ctx == 1 assert service.ctx == 2
def test_context_multiple_set(): results = [] async def test(): context = aiomisc.get_context() context["foo"] = True await asyncio.sleep(0.1) results.append(await context["foo"]) context["foo"] = False await asyncio.sleep(0.1) results.append(await context["foo"]) context["foo"] = None await asyncio.sleep(0.1) results.append(await context["foo"]) with aiomisc.entrypoint() as loop: loop.run_until_complete(asyncio.wait_for(test(), timeout=10), ) assert results == [True, False, None]
def test_tcp_client(aiomisc_socket_factory, localhost): class TestService(TCPServer): DATA = [] async def handle_client( self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, ): self.DATA.append(await reader.readline()) class TestClient(TCPClient): event: asyncio.Event async def handle_connection( self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, ) -> None: writer.write(b"hello server\n") await writer.drain() self.loop.call_soon(self.event.set) port, sock = aiomisc_socket_factory() event = asyncio.Event() services = [ TestService(sock=sock), TestClient(address=localhost, port=port, event=event), ] async def go(): await event.wait() with aiomisc.entrypoint(*services) as loop: loop.run_until_complete(asyncio.wait_for(go(), timeout=10), ) assert TestService.DATA assert TestService.DATA == [b"hello server\n"]
def test_shield(): results = [] @aiomisc.shield async def coro(): nonlocal results await asyncio.sleep(0.5) results.append(True) async def main(loop): task = loop.create_task(coro()) task.cancel() try: await task except asyncio.CancelledError: pass finally: await asyncio.sleep(1) with aiomisc.entrypoint() as loop: loop.run_until_complete(main(loop)) assert results == [True]
def test_respawning_process_service(tmpdir): queue = Queue() svc = SimpleRespawningProcessService( queue=queue, process_poll_timeout=0.5, ) @timeout(5) async def go(): pids = [] @threaded def getter(): return queue.get() for _ in range(2): pids.append(await getter()) assert len(pids) == 2 assert pids[0] != pids[1] with aiomisc.entrypoint(svc) as loop: loop.run_until_complete(go())
def test_udp_server(aiomisc_unused_port): class TestService(UDPServer): DATA = [] async def handle_datagram(self, data: bytes, addr: tuple): self.DATA.append(data) service = TestService("127.0.0.1", aiomisc_unused_port) @aiomisc.threaded def writer(): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) with sock: sock.sendto(b"hello server\n", ("127.0.0.1", aiomisc_unused_port)) with aiomisc.entrypoint(service) as loop: loop.run_until_complete( asyncio.wait_for(writer(), timeout=10), ) assert TestService.DATA assert TestService.DATA == [b"hello server\n"]
def test_tls_server(certs, ssl_client_context, aiomisc_unused_port): class TestService(TLSServer): DATA = [] async def handle_client(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): self.DATA.append(await reader.readline()) writer.close() service = TestService( address="127.0.0.1", port=aiomisc_unused_port, ca=certs / "ca.pem", key=certs / "server.key", cert=certs / "server.pem", ) @aiomisc.threaded def writer(): with ExitStack() as stack: sock = stack.enter_context( socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0), ) ssock = stack.enter_context( ssl_client_context.wrap_socket( sock, server_hostname="localhost", ), ) ssock.connect(("127.0.0.1", aiomisc_unused_port)) ssock.send(b"hello server\n") with aiomisc.entrypoint(service) as loop: loop.run_until_complete(asyncio.wait_for(writer(), timeout=10), ) assert TestService.DATA assert TestService.DATA == [b"hello server\n"]
def test_coroutine_function_dependency(): @dependency async def foo(): await asyncio.sleep(0.1) return 'Foo' @dependency async def bar(): return 'Bar' class TestService(Service): __dependencies__ = ( 'foo', 'bar', ) async def start(self): ... service = TestService() with entrypoint(service): assert service.foo == 'Foo' assert service.bar == 'Bar'
def process(self): with aiomisc.entrypoint() as loop: self.loop = loop try: self._process() except Exception as e: log.error('GTAFieldParser failed') if production: log.info('Sending e-mail to developers') stacktrace = traceback.format_exc() text = f'<pre>{stacktrace}</pre>' staff_emails = get_user_model().staff_emails() msg = EmailMessage( subject='Reporter: GTAFieldParser failure', body=text, from_email=None, to=staff_emails, cc=['*****@*****.**'], ) msg.content_subtype = 'html' msg.send() raise e finally: self.loop = None
"application/dns-message", aiohttp.hdrs.CONTENT_TYPE: "application/dns-message", }) async with request as response: log.debug("Handling request from %r", addr) self.sendto(await response.read(), addr) async def start(self) -> None: await super().start() self.session = aiohttp.ClientSession() async def stop(self, exc: Exception = None) -> None: await self.session.close() await super().stop(exc) if __name__ == '__main__': parser = Parser(auto_env_var_prefix="DOH_") parser.parse_args() services = [ DNSUDPService(address=parser.address, port=parser.port, url=parser.doh_url), SDWatchdogService(), ] with aiomisc.entrypoint(*services, log_level=parser.log_level) as loop: loop.run_forever()
while True: await asyncio.gather(*[p.pull() for p in self.pullers]) now = datetime.now() await asyncio.gather(*[ ps.push(pl.stats, now=now, provider=str(pl)) for pl in self.pullers for ps in self.pushers ]) await asyncio.sleep(self._config['update_interval']) async def main(): config = yaml.safe_load(open('config.yml', 'r')) logging_config = yaml.safe_load(open('logging.yml', 'r')) logging_config['root']['level'] = config['log_level'] logging.config.dictConfig(logging_config) logging.debug('Config: {}'.format(pformat(config))) db_prefix = config['db_prefix'] if not os.path.exists(f'./{db_prefix}'): os.mkdir(db_prefix) mb = Moneybot(config) await mb.flow() with aiomisc.entrypoint() as loop: loop.run_until_complete(main())
def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( "--debug", action="store_true", help="Debug mode (default False)" ) parser.add_argument( "--extra-search-path", action="store_true", help=( "Try harder to find the 'libpython*' shared library at the cost " "of a slower server startup. (default False)" ), ) parser.add_argument( "--more", action="store_true", help="Set the debug more verbose (default False)", ) parser.add_argument( "--detached_session", action="store_true", help="Whether to continue program on browser close (default False)", ) parser.add_argument( "--show-filename", action="store_true", help="Whether to show filename in session list (default False)", ) parser.add_argument( "--server-host", type=str, default="localhost", help="Host used to serve debugging pages (default localhost)", ) parser.add_argument( "--server-port", type=int, default=1984, help="Port used to serve debugging pages (default 1984)", ) parser.add_argument( "--socket-host", type=str, default="localhost", help="Host used to communicate with wdb instances (default localhost)", ) parser.add_argument( "--socket-port", type=int, default=19840, help="Port used to communicate with wdb instances (default 19840)", ) args = parser.parse_args() log = getLogger("wdb_server") if args.debug: log.setLevel(INFO) if args.more: log.setLevel(DEBUG) else: log.setLevel(WARNING) uvloop.install() services = ( WDBService( address=args.server_host, port=args.server_port, **vars(args), ), WDBTCPService(address=args.socket_host, port=args.socket_port), ) with entrypoint(*services) as loop: try: loop.run_forever() except KeyboardInterrupt: print("Received exit, exiting")
def test_journald_logger(loop, subtests): with TemporaryDirectory(dir="/tmp") as tmp_dir: tmp_path = Path(tmp_dir) sock_path = tmp_path / "notify.sock" logs = deque() class FakeJournald(UDPServer): VALUE_LEN_STRUCT = struct.Struct("<Q") def handle_datagram(self, data: bytes, addr: tuple) -> None: result = {} with BytesIO(data) as fp: line = fp.readline() while line: if b"=" not in line: key = line.decode().strip() value_len = self.VALUE_LEN_STRUCT.unpack( fp.read( self.VALUE_LEN_STRUCT.size, ), )[0] value = fp.read(value_len).decode() assert fp.read(1) == b"\n" else: key, value = map( lambda x: x.strip(), line.decode().split("=", 1), ) result[key] = value line = fp.readline() logs.append(result) @threaded def log_writer(): log = logging.getLogger("test") log.propagate = False log.handlers.clear() log.handlers.append(JournaldLogHandler()) log.info("Test message") log.info("Test multiline\nmessage") log.info( "Test formatted: int=%d str=%s repr=%r float=%0.1f", 1, 2, 3, 4, ) log.info( "Message with extra", extra={ "foo": "bar", }, ) try: 1 / 0 except ZeroDivisionError: log.exception("Sample exception") with bind_socket( socket.AF_UNIX, socket.SOCK_DGRAM, address=str(sock_path), ) as sock: JournaldLogHandler.SOCKET_PATH = sock_path with aiomisc.entrypoint(FakeJournald(sock=sock), loop=loop): loop.run_until_complete(log_writer()) assert len(logs) == 5 required_fields = { "MESSAGE", "MESSAGE_ID", "MESSAGE_RAW", "PRIORITY", "SYSLOG_FACILITY", "CODE", "CODE_FUNC", "CODE_FILE", "CODE_LINE", "CODE_MODULE", "LOGGER_NAME", "PID", "PROCESS_NAME", "THREAD_ID", "THREAD_NAME", "RELATIVE_USEC", "CREATED_USEC", } with subtests.test("simple message"): message = logs.popleft() assert message["MESSAGE"] == "Test message" assert message["MESSAGE_RAW"] == "Test message" assert message["PRIORITY"] == "6" assert message["CODE_FUNC"] == "log_writer" assert int(message["PID"]) == os.getpid() for field in required_fields: assert field in message with subtests.test("multiline message"): message = logs.popleft() assert message["MESSAGE"] == "Test multiline\nmessage" assert message["MESSAGE_RAW"] == "Test multiline\nmessage" assert message["PRIORITY"] == "6" assert message["CODE_FUNC"] == "log_writer" assert int(message["PID"]) == os.getpid() for field in required_fields: assert field in message with subtests.test("formatted message"): message = logs.popleft() assert message["MESSAGE"] == ( "Test formatted: int=1 str=2 repr=3 float=4.0" ) assert message["MESSAGE_RAW"] == ( "Test formatted: int=%d str=%s repr=%r float=%0.1f" ) assert message["ARGUMENTS_0"] == "1" assert message["ARGUMENTS_1"] == "2" assert message["ARGUMENTS_2"] == "3" assert message["ARGUMENTS_3"] == "4" assert message["PRIORITY"] == "6" assert message["CODE_FUNC"] == "log_writer" assert int(message["PID"]) == os.getpid() for field in required_fields: assert field in message with subtests.test("message with extra"): message = logs.popleft() assert message["MESSAGE"] == "Message with extra" assert message["MESSAGE_RAW"] == "Message with extra" assert message["PRIORITY"] == "6" assert message["CODE_FUNC"] == "log_writer" assert message["EXTRA_FOO"] == "bar" assert int(message["PID"]) == os.getpid() for field in required_fields: assert field in message with subtests.test("exception message"): message = logs.popleft() assert message["MESSAGE"].startswith("Sample exception\nTraceback") assert message["MESSAGE_RAW"] == "Sample exception" assert message["PRIORITY"] == "3" assert message["CODE_FUNC"] == "log_writer" assert int(message["PID"]) == os.getpid() assert message["EXCEPTION_TYPE"] == "<class 'ZeroDivisionError'>" assert message["EXCEPTION_VALUE"] == "division by zero" assert message["TRACEBACK"].startswith( "Traceback (most recent call last)", ) for field in required_fields: assert field in message
env.result_queue.put_nowait( loop.run_in_executor(env.executor, cpu_bound, t)) services = ( REST(address=env.config['address'], port=env.config['port']), ConsumerService(), SaveResultService(), ) if __name__ == '__main__': log = logging.getLogger('Service') loop = asyncio.get_event_loop() # очередь невыполненных задач env.task_queue = asyncio.Queue(loop=loop) # очередь запущенных или отработанных задач, результат которых не сохранён env.result_queue = asyncio.Queue(loop=loop) # устанавливаем максимальное количество воркеров для cpu bound задач из конфига или по числу ядер max_workers = env.config.get('max_workers', os.cpu_count()) # cpu bound запускаем в отдельном процессе env.executor = concurrent.futures.ProcessPoolExecutor( max_workers=max_workers, ) env.results = {} with entrypoint(*services, loop=loop) as loop: loop.run_forever()
auto_env_var_prefix="SERVER_", description="Async HTTP server", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ignore_unknown_config_file_keys=True, ) parser.add_argument( "-s", "--pool-size", type=int, default=4, help="Thread pool size" ) parser.add_argument( "--log-level", default="info", choices=("debug", "info", "warning", "error", "fatal"), help="Logging options" ) group = parser.add_argument_group("API Options") group.add_argument("--address", default="127.0.0.1") group.add_argument("--port", type=int, default=80) if __name__ == "__main__": arguments = parser.parse_args() environ.clear() services = [ HTTPServer(address=arguments.address, port=arguments.port) ] with entrypoint( *services, log_level=arguments.log_level, pool_size=arguments.pool_size) as loop: loop.run_forever()
) async def start_background_tasks(app): app['rate_updater'] = asyncio.create_task(update_rate(app)) log.info("Background task are created") async def cleanup_background_tasks(app): app['rate_updater'].cancel() log.info("Background task are cancelled") await app['rate_updater'] class ExchangeRate(AIOHTTPService): async def create_application(self): app = web.Application() setup_routes(app) app.on_startup.append(start_background_tasks) app.on_cleanup.append(cleanup_background_tasks) log.info("Instance of app is created") return app service = ExchangeRate(address="127.0.0.1", port=8080) with entrypoint(service) as loop: log.info("Starting loop") loop.run_forever() log.info("Stopped")
import sentry_sdk from aiomisc import entrypoint, receiver from sentry_sdk.integrations.aiohttp import AioHttpIntegration from .config import app_config from .misc.hooks import init_db, close_db from .services import services if app_config.sentry_dsn: sentry_sdk.init(app_config.sentry_dsn, release=app_config.release, integrations=[AioHttpIntegration()]) @receiver(entrypoint.PRE_START) async def start_up(*args, **kwargs): await init_db() @receiver(entrypoint.POST_STOP) async def shutdown(*args, **kwargs): await close_db() if __name__ == '__main__': with entrypoint(*services) as loop: try: loop.run_forever() except (KeyboardInterrupt, SystemExit): pass
def loop(self): with entrypoint() as loop: yield loop
def loop(): with entrypoint() as loop: asyncio.set_event_loop(loop) yield loop