def test_tcp_server_unix(unix_socket_tcp):
class TestService(TCPServer):
DATA = []
async def handle_client(self, reader: asyncio.StreamReader,
writer: asyncio.StreamWriter):
self.DATA.append(await reader.readline())
writer.close()
service = TestService(sock=unix_socket_tcp)
@aiomisc.threaded
def writer():
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
sock.connect(unix_socket_tcp.getsockname())
sock.send(b"hello server\n")
with aiomisc.entrypoint(service) as loop:
loop.run_until_complete(asyncio.wait_for(writer(), timeout=10), )
assert TestService.DATA
assert TestService.DATA == [b"hello server\n"]
def test_service_events():
class Initialization(aiomisc.Service):
async def start(self):
context = aiomisc.get_context()
await asyncio.sleep(0.1)
context["test"] = True
class Awaiter(aiomisc.Service):
result = None
async def start(self):
Awaiter.result = await self.context["test"]
services = (
Awaiter(),
Initialization(),
)
with aiomisc.entrypoint(*services):
pass
assert Awaiter.result
def test_tcp_server(aiomisc_unused_port):
class TestService(TCPServer):
DATA = []
async def handle_client(self, reader: asyncio.StreamReader,
writer: asyncio.StreamWriter):
self.DATA.append(await reader.readline())
writer.close()
service = TestService("127.0.0.1", aiomisc_unused_port)
@aiomisc.threaded
def writer():
port = aiomisc_unused_port
with socket.create_connection(("127.0.0.1", port)) as sock:
sock.send(b"hello server\n")
with aiomisc.entrypoint(service) as loop:
loop.run_until_complete(asyncio.wait_for(writer(), timeout=10), )
assert TestService.DATA
assert TestService.DATA == [b"hello server\n"]
async def test_entrypoint_with_with_async():
class MyService(aiomisc.service.Service):
ctx = 0
async def start(self):
self.__class__.ctx = 1
async def stop(self, exc: Exception = None):
self.__class__.ctx = 2
service = MyService()
assert service.ctx == 0
async with aiomisc.entrypoint(service) as ep:
assert service.ctx == 1
with pytest.raises(asyncio.TimeoutError):
await asyncio.wait_for(ep.closing(), timeout=0.3)
assert service.ctx == 1
assert service.ctx == 2
def test_context_multiple_set():
results = []
async def test():
context = aiomisc.get_context()
context["foo"] = True
await asyncio.sleep(0.1)
results.append(await context["foo"])
context["foo"] = False
await asyncio.sleep(0.1)
results.append(await context["foo"])
context["foo"] = None
await asyncio.sleep(0.1)
results.append(await context["foo"])
with aiomisc.entrypoint() as loop:
loop.run_until_complete(asyncio.wait_for(test(), timeout=10), )
assert results == [True, False, None]
def test_tcp_client(aiomisc_socket_factory, localhost):
class TestService(TCPServer):
DATA = []
async def handle_client(
self,
reader: asyncio.StreamReader,
writer: asyncio.StreamWriter,
):
self.DATA.append(await reader.readline())
class TestClient(TCPClient):
event: asyncio.Event
async def handle_connection(
self,
reader: asyncio.StreamReader,
writer: asyncio.StreamWriter,
) -> None:
writer.write(b"hello server\n")
await writer.drain()
self.loop.call_soon(self.event.set)
port, sock = aiomisc_socket_factory()
event = asyncio.Event()
services = [
TestService(sock=sock),
TestClient(address=localhost, port=port, event=event),
]
async def go():
await event.wait()
with aiomisc.entrypoint(*services) as loop:
loop.run_until_complete(asyncio.wait_for(go(), timeout=10), )
assert TestService.DATA
assert TestService.DATA == [b"hello server\n"]
def test_shield():
results = []
@aiomisc.shield
async def coro():
nonlocal results
await asyncio.sleep(0.5)
results.append(True)
async def main(loop):
task = loop.create_task(coro())
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
finally:
await asyncio.sleep(1)
with aiomisc.entrypoint() as loop:
loop.run_until_complete(main(loop))
assert results == [True]
def test_respawning_process_service(tmpdir):
queue = Queue()
svc = SimpleRespawningProcessService(
queue=queue,
process_poll_timeout=0.5,
)
@timeout(5)
async def go():
pids = []
@threaded
def getter():
return queue.get()
for _ in range(2):
pids.append(await getter())
assert len(pids) == 2
assert pids[0] != pids[1]
with aiomisc.entrypoint(svc) as loop:
loop.run_until_complete(go())
def test_udp_server(aiomisc_unused_port):
class TestService(UDPServer):
DATA = []
async def handle_datagram(self, data: bytes, addr: tuple):
self.DATA.append(data)
service = TestService("127.0.0.1", aiomisc_unused_port)
@aiomisc.threaded
def writer():
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
with sock:
sock.sendto(b"hello server\n", ("127.0.0.1", aiomisc_unused_port))
with aiomisc.entrypoint(service) as loop:
loop.run_until_complete(
asyncio.wait_for(writer(), timeout=10),
)
assert TestService.DATA
assert TestService.DATA == [b"hello server\n"]
def test_tls_server(certs, ssl_client_context, aiomisc_unused_port):
class TestService(TLSServer):
DATA = []
async def handle_client(self, reader: asyncio.StreamReader,
writer: asyncio.StreamWriter):
self.DATA.append(await reader.readline())
writer.close()
service = TestService(
address="127.0.0.1",
port=aiomisc_unused_port,
ca=certs / "ca.pem",
key=certs / "server.key",
cert=certs / "server.pem",
)
@aiomisc.threaded
def writer():
with ExitStack() as stack:
sock = stack.enter_context(
socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0), )
ssock = stack.enter_context(
ssl_client_context.wrap_socket(
sock,
server_hostname="localhost",
), )
ssock.connect(("127.0.0.1", aiomisc_unused_port))
ssock.send(b"hello server\n")
with aiomisc.entrypoint(service) as loop:
loop.run_until_complete(asyncio.wait_for(writer(), timeout=10), )
assert TestService.DATA
assert TestService.DATA == [b"hello server\n"]
def test_coroutine_function_dependency():
@dependency
async def foo():
await asyncio.sleep(0.1)
return 'Foo'
@dependency
async def bar():
return 'Bar'
class TestService(Service):
__dependencies__ = (
'foo',
'bar',
)
async def start(self):
...
service = TestService()
with entrypoint(service):
assert service.foo == 'Foo'
assert service.bar == 'Bar'
def process(self):
with aiomisc.entrypoint() as loop:
self.loop = loop
try:
self._process()
except Exception as e:
log.error('GTAFieldParser failed')
if production:
log.info('Sending e-mail to developers')
stacktrace = traceback.format_exc()
text = f'<pre>{stacktrace}</pre>'
staff_emails = get_user_model().staff_emails()
msg = EmailMessage(
subject='Reporter: GTAFieldParser failure',
body=text,
from_email=None,
to=staff_emails,
cc=['*****@*****.**'],
)
msg.content_subtype = 'html'
msg.send()
raise e
finally:
self.loop = None
"application/dns-message",
aiohttp.hdrs.CONTENT_TYPE:
"application/dns-message",
})
async with request as response:
log.debug("Handling request from %r", addr)
self.sendto(await response.read(), addr)
async def start(self) -> None:
await super().start()
self.session = aiohttp.ClientSession()
async def stop(self, exc: Exception = None) -> None:
await self.session.close()
await super().stop(exc)
if __name__ == '__main__':
parser = Parser(auto_env_var_prefix="DOH_")
parser.parse_args()
services = [
DNSUDPService(address=parser.address,
port=parser.port,
url=parser.doh_url),
SDWatchdogService(),
]
with aiomisc.entrypoint(*services, log_level=parser.log_level) as loop:
loop.run_forever()
while True:
await asyncio.gather(*[p.pull() for p in self.pullers])
now = datetime.now()
await asyncio.gather(*[
ps.push(pl.stats, now=now, provider=str(pl))
for pl in self.pullers for ps in self.pushers
])
await asyncio.sleep(self._config['update_interval'])
async def main():
config = yaml.safe_load(open('config.yml', 'r'))
logging_config = yaml.safe_load(open('logging.yml', 'r'))
logging_config['root']['level'] = config['log_level']
logging.config.dictConfig(logging_config)
logging.debug('Config: {}'.format(pformat(config)))
db_prefix = config['db_prefix']
if not os.path.exists(f'./{db_prefix}'):
os.mkdir(db_prefix)
mb = Moneybot(config)
await mb.flow()
with aiomisc.entrypoint() as loop:
loop.run_until_complete(main())
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument(
"--debug", action="store_true", help="Debug mode (default False)"
)
parser.add_argument(
"--extra-search-path",
action="store_true",
help=(
"Try harder to find the 'libpython*' shared library at the cost "
"of a slower server startup. (default False)"
),
)
parser.add_argument(
"--more",
action="store_true",
help="Set the debug more verbose (default False)",
)
parser.add_argument(
"--detached_session",
action="store_true",
help="Whether to continue program on browser close (default False)",
)
parser.add_argument(
"--show-filename",
action="store_true",
help="Whether to show filename in session list (default False)",
)
parser.add_argument(
"--server-host",
type=str,
default="localhost",
help="Host used to serve debugging pages (default localhost)",
)
parser.add_argument(
"--server-port",
type=int,
default=1984,
help="Port used to serve debugging pages (default 1984)",
)
parser.add_argument(
"--socket-host",
type=str,
default="localhost",
help="Host used to communicate with wdb instances (default localhost)",
)
parser.add_argument(
"--socket-port",
type=int,
default=19840,
help="Port used to communicate with wdb instances (default 19840)",
)
args = parser.parse_args()
log = getLogger("wdb_server")
if args.debug:
log.setLevel(INFO)
if args.more:
log.setLevel(DEBUG)
else:
log.setLevel(WARNING)
uvloop.install()
services = (
WDBService(
address=args.server_host,
port=args.server_port,
**vars(args),
),
WDBTCPService(address=args.socket_host, port=args.socket_port),
)
with entrypoint(*services) as loop:
try:
loop.run_forever()
except KeyboardInterrupt:
print("Received exit, exiting")
def test_journald_logger(loop, subtests):
with TemporaryDirectory(dir="/tmp") as tmp_dir:
tmp_path = Path(tmp_dir)
sock_path = tmp_path / "notify.sock"
logs = deque()
class FakeJournald(UDPServer):
VALUE_LEN_STRUCT = struct.Struct("<Q")
def handle_datagram(self, data: bytes, addr: tuple) -> None:
result = {}
with BytesIO(data) as fp:
line = fp.readline()
while line:
if b"=" not in line:
key = line.decode().strip()
value_len = self.VALUE_LEN_STRUCT.unpack(
fp.read(
self.VALUE_LEN_STRUCT.size,
),
)[0]
value = fp.read(value_len).decode()
assert fp.read(1) == b"\n"
else:
key, value = map(
lambda x: x.strip(),
line.decode().split("=", 1),
)
result[key] = value
line = fp.readline()
logs.append(result)
@threaded
def log_writer():
log = logging.getLogger("test")
log.propagate = False
log.handlers.clear()
log.handlers.append(JournaldLogHandler())
log.info("Test message")
log.info("Test multiline\nmessage")
log.info(
"Test formatted: int=%d str=%s repr=%r float=%0.1f",
1, 2, 3, 4,
)
log.info(
"Message with extra", extra={
"foo": "bar",
},
)
try:
1 / 0
except ZeroDivisionError:
log.exception("Sample exception")
with bind_socket(
socket.AF_UNIX, socket.SOCK_DGRAM, address=str(sock_path),
) as sock:
JournaldLogHandler.SOCKET_PATH = sock_path
with aiomisc.entrypoint(FakeJournald(sock=sock), loop=loop):
loop.run_until_complete(log_writer())
assert len(logs) == 5
required_fields = {
"MESSAGE", "MESSAGE_ID", "MESSAGE_RAW", "PRIORITY",
"SYSLOG_FACILITY", "CODE", "CODE_FUNC", "CODE_FILE",
"CODE_LINE", "CODE_MODULE", "LOGGER_NAME", "PID",
"PROCESS_NAME", "THREAD_ID", "THREAD_NAME",
"RELATIVE_USEC", "CREATED_USEC",
}
with subtests.test("simple message"):
message = logs.popleft()
assert message["MESSAGE"] == "Test message"
assert message["MESSAGE_RAW"] == "Test message"
assert message["PRIORITY"] == "6"
assert message["CODE_FUNC"] == "log_writer"
assert int(message["PID"]) == os.getpid()
for field in required_fields:
assert field in message
with subtests.test("multiline message"):
message = logs.popleft()
assert message["MESSAGE"] == "Test multiline\nmessage"
assert message["MESSAGE_RAW"] == "Test multiline\nmessage"
assert message["PRIORITY"] == "6"
assert message["CODE_FUNC"] == "log_writer"
assert int(message["PID"]) == os.getpid()
for field in required_fields:
assert field in message
with subtests.test("formatted message"):
message = logs.popleft()
assert message["MESSAGE"] == (
"Test formatted: int=1 str=2 repr=3 float=4.0"
)
assert message["MESSAGE_RAW"] == (
"Test formatted: int=%d str=%s repr=%r float=%0.1f"
)
assert message["ARGUMENTS_0"] == "1"
assert message["ARGUMENTS_1"] == "2"
assert message["ARGUMENTS_2"] == "3"
assert message["ARGUMENTS_3"] == "4"
assert message["PRIORITY"] == "6"
assert message["CODE_FUNC"] == "log_writer"
assert int(message["PID"]) == os.getpid()
for field in required_fields:
assert field in message
with subtests.test("message with extra"):
message = logs.popleft()
assert message["MESSAGE"] == "Message with extra"
assert message["MESSAGE_RAW"] == "Message with extra"
assert message["PRIORITY"] == "6"
assert message["CODE_FUNC"] == "log_writer"
assert message["EXTRA_FOO"] == "bar"
assert int(message["PID"]) == os.getpid()
for field in required_fields:
assert field in message
with subtests.test("exception message"):
message = logs.popleft()
assert message["MESSAGE"].startswith("Sample exception\nTraceback")
assert message["MESSAGE_RAW"] == "Sample exception"
assert message["PRIORITY"] == "3"
assert message["CODE_FUNC"] == "log_writer"
assert int(message["PID"]) == os.getpid()
assert message["EXCEPTION_TYPE"] == "<class 'ZeroDivisionError'>"
assert message["EXCEPTION_VALUE"] == "division by zero"
assert message["TRACEBACK"].startswith(
"Traceback (most recent call last)",
)
for field in required_fields:
assert field in message
env.result_queue.put_nowait(
loop.run_in_executor(env.executor, cpu_bound, t))
services = (
REST(address=env.config['address'], port=env.config['port']),
ConsumerService(),
SaveResultService(),
)
if __name__ == '__main__':
log = logging.getLogger('Service')
loop = asyncio.get_event_loop()
# очередь невыполненных задач
env.task_queue = asyncio.Queue(loop=loop)
# очередь запущенных или отработанных задач, результат которых не сохранён
env.result_queue = asyncio.Queue(loop=loop)
# устанавливаем максимальное количество воркеров для cpu bound задач из конфига или по числу ядер
max_workers = env.config.get('max_workers', os.cpu_count())
# cpu bound запускаем в отдельном процессе
env.executor = concurrent.futures.ProcessPoolExecutor(
max_workers=max_workers, )
env.results = {}
with entrypoint(*services, loop=loop) as loop:
loop.run_forever()
auto_env_var_prefix="SERVER_",
description="Async HTTP server",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
ignore_unknown_config_file_keys=True,
)
parser.add_argument(
"-s", "--pool-size", type=int, default=4, help="Thread pool size"
)
parser.add_argument(
"--log-level",
default="info",
choices=("debug", "info", "warning", "error", "fatal"),
help="Logging options"
)
group = parser.add_argument_group("API Options")
group.add_argument("--address", default="127.0.0.1")
group.add_argument("--port", type=int, default=80)
if __name__ == "__main__":
arguments = parser.parse_args()
environ.clear()
services = [
HTTPServer(address=arguments.address, port=arguments.port)
]
with entrypoint(
*services,
log_level=arguments.log_level,
pool_size=arguments.pool_size) as loop:
loop.run_forever()
)
async def start_background_tasks(app):
app['rate_updater'] = asyncio.create_task(update_rate(app))
log.info("Background task are created")
async def cleanup_background_tasks(app):
app['rate_updater'].cancel()
log.info("Background task are cancelled")
await app['rate_updater']
class ExchangeRate(AIOHTTPService):
async def create_application(self):
app = web.Application()
setup_routes(app)
app.on_startup.append(start_background_tasks)
app.on_cleanup.append(cleanup_background_tasks)
log.info("Instance of app is created")
return app
service = ExchangeRate(address="127.0.0.1", port=8080)
with entrypoint(service) as loop:
log.info("Starting loop")
loop.run_forever()
log.info("Stopped")
import sentry_sdk
from aiomisc import entrypoint, receiver
from sentry_sdk.integrations.aiohttp import AioHttpIntegration
from .config import app_config
from .misc.hooks import init_db, close_db
from .services import services
if app_config.sentry_dsn:
sentry_sdk.init(app_config.sentry_dsn, release=app_config.release, integrations=[AioHttpIntegration()])
@receiver(entrypoint.PRE_START)
async def start_up(*args, **kwargs):
await init_db()
@receiver(entrypoint.POST_STOP)
async def shutdown(*args, **kwargs):
await close_db()
if __name__ == '__main__':
with entrypoint(*services) as loop:
try:
loop.run_forever()
except (KeyboardInterrupt, SystemExit):
pass
def loop(self):
with entrypoint() as loop:
yield loop
def loop():
with entrypoint() as loop:
asyncio.set_event_loop(loop)
yield loop
