def test_request_without_subscriber_throws(runner, ipc_base_path):
server_config = ConnectionConfig.from_name("server", base_path=ipc_base_path)
server = d.driver(d.serve_endpoint(server_config))
client = d.driver(
d.run_endpoint("client"), d.throws(d.request(Request()), NoSubscribers)
)
runner(server, client)
async def run_network(self):
async with AsyncioEndpoint.serve(
ConnectionConfig.from_name("network")) as self.network_bus:
self.logger.info("Network started")
self.network_bus.subscribe(GenericMessageOutgoingEvent,
self.send_to_all_nodes)
self.logger.info("Network subscribed to all messages")
while not self.kill_switch.value:
await asyncio.sleep(0)
self.logger.info("Network turned off")
async def worker(backend: BaseBackend, name: str, num_events: int) -> None:
config = ConnectionConfig.from_name(name)
async with backend.Endpoint.serve(config) as event_bus:
await event_bus.connect_to_endpoints(
ConnectionConfig.from_name(REPORTER_ENDPOINT)
)
await event_bus.wait_until_all_remotes_subscribed_to(TotalRecordedEvent)
stats = LocalStatistic()
events = event_bus.stream(PerfMeasureEvent, num_events=num_events)
async for event in events:
stats.add(
RawMeasureEntry(sent_at=event.sent_at, received_at=time.time())
)
await event_bus.broadcast(
TotalRecordedEvent(stats.crunch(event_bus.name)),
BroadcastConfig(filter_endpoint=REPORTER_ENDPOINT),
)
def __init__(self,
trinity_config: TrinityConfig,
endpoint_name: str,
cancel_token: CancelToken = None,
loop: asyncio.AbstractEventLoop = None) -> None:
self._trinity_config = trinity_config
self._endpoint_available = asyncio.Event()
self._connection_config = ConnectionConfig.from_name(
endpoint_name, self._trinity_config.ipc_dir)
super().__init__(cancel_token, loop)
async def test_base_wait_until_connections_changed():
config = ConnectionConfig.from_name(generate_unique_name())
async with AsyncioEndpoint.serve(config):
async with AsyncioEndpoint("client").run() as client:
asyncio.ensure_future(client.connect_to_endpoints(config))
assert not client.is_connected_to(config.name)
await asyncio.wait_for(client.wait_until_connections_change(),
timeout=0.1)
assert client.is_connected_to(config.name)
async def proc2_worker():
config = ConnectionConfig.from_name("e1")
async with AsyncioEndpoint("e2").run() as client:
await client.connect_to_endpoints(config)
await client.wait_until_any_endpoint_subscribed_to(GetSomethingRequest)
for i in range(3):
print("Requesting")
result = await client.request(GetSomethingRequest())
print(f"Got answer: {result.payload}")
async def run(self) -> None:
connection_config = ConnectionConfig.from_name(
MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir)
async with AsyncioEndpoint.serve(connection_config) as endpoint:
self._endpoint = endpoint
# start the background process that tracks and propagates available
# endpoints to the other connected endpoints
self.manager.run_daemon_task(
self._track_and_propagate_available_endpoints)
await endpoint.wait_until_any_endpoint_subscribed_to(
EventBusConnected)
# signal the endpoint is up and running and available
self._endpoint_available.set()
# instantiate all of the components
all_components = tuple(
component_cls(self._boot_info)
for component_cls in self._component_types)
# filter out any components that should not be enabled.
enabled_components = tuple(component
for component in all_components
if component.is_enabled)
# a little bit of extra try/finally structure here to produce good
# logging messages about the component lifecycle.
from p2p.asyncio_utils import create_task, wait_first
try:
self.logger.info(
"Starting components: %s",
'/'.join(component.name
for component in enabled_components),
)
tasks: List[asyncio.Task[Any]] = []
for component in enabled_components:
tasks.append(
create_task(
component.run_in_process(),
f'IsolatedComponent/{component.name}/run_in_process'
))
tasks.append(
asyncio.create_task(self._trigger_component_exit.wait()))
self.logger.info("Components started")
try:
# The timeout is long as our component tasks can do a lot of stuff during
# their cleanup.
await wait_first(tasks, max_wait_after_cancellation=10)
finally:
self.logger.info("Stopping components")
finally:
self.logger.info("Components stopped.")
self.manager.cancel()
async def run(self) -> None:
connection_config = ConnectionConfig.from_name(
MAIN_EVENTBUS_ENDPOINT,
self._boot_info.trinity_config.ipc_dir
)
async with AsyncioEndpoint.serve(connection_config) as endpoint:
self._endpoint = endpoint
# start the background process that tracks and propagates available
# endpoints to the other connected endpoints
self.manager.run_daemon_task(self._track_and_propagate_available_endpoints)
self.manager.run_daemon_task(self._handle_shutdown_request)
await endpoint.wait_until_any_endpoint_subscribed_to(ShutdownRequest)
await endpoint.wait_until_any_endpoint_subscribed_to(EventBusConnected)
# signal the endpoint is up and running and available
self._endpoint_available.set()
# instantiate all of the components
all_components = tuple(
component_cls(self._boot_info)
for component_cls
in self._component_types
)
# filter out any components that should not be enabled.
enabled_components = tuple(
component
for component in all_components
if component.is_enabled
)
# a little bit of extra try/finally structure here to produce good
# logging messages about the component lifecycle.
try:
async with AsyncExitStack() as stack:
self.logger.info(
"Starting components: %s",
'/'.join(component.name for component in enabled_components),
)
# Concurrently start the components.
await asyncio.gather(*(
stack.enter_async_context(run_component(component))
for component in enabled_components
))
self.logger.info("Components started")
try:
await self._trigger_component_exit.wait()
finally:
self.logger.info("Stopping components")
finally:
self.logger.info("Components stopped.")
self.manager.cancel()
async def test_endpoint_serve_with_error(ipc_base_path):
config = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path)
with pytest.raises(Exception, match="break out of serve"):
async with AsyncioEndpoint.serve(config) as endpoint:
assert endpoint.is_running is True
assert endpoint.is_serving is True
raise Exception("break out of serve")
assert endpoint.is_running is False
assert endpoint.is_serving is False
async def run(self) -> None:
connection_config = ConnectionConfig.from_name(
MAIN_EVENTBUS_ENDPOINT,
self._boot_info.trinity_config.ipc_dir
)
async with AsyncioEndpoint.serve(connection_config) as endpoint:
self._endpoint = endpoint
# start the background process that tracks and propagates available
# endpoints to the other connected endpoints
self.manager.run_daemon_task(self._track_and_propagate_available_endpoints)
await endpoint.wait_until_any_endpoint_subscribed_to(EventBusConnected)
# signal the endpoint is up and running and available
self._endpoint_available.set()
# instantiate all of the components
all_components = tuple(
component_cls(self._boot_info)
for component_cls
in self._component_types
)
# filter out any components that should not be enabled.
enabled_components = tuple(
component
for component in all_components
if component.is_enabled
)
# a little bit of extra try/finally structure here to produce good
# logging messages about the component lifecycle.
from p2p.logic import wait_first
try:
self.logger.info(
"Starting components: %s",
'/'.join(component.name for component in enabled_components),
)
context_managers = [component.run() for component in enabled_components]
async with AsyncContextGroup(context_managers) as futs:
# AsyncContextGroup() yields a Sequence[Any], so we cast to a list of Futures
# here to ensure mypy can come to our aid if we forget the create_task() when
# adding new entries to the list of Futures we want to wait for.
futures = cast(List["asyncio.Future[None]"], list(futs))
futures.append(asyncio.create_task(self._trigger_component_exit.wait()))
self.logger.info("Components started")
try:
await wait_first(futures)
finally:
self.logger.info("Stopping components")
finally:
self.logger.info("Components stopped.")
self.manager.cancel()
async def _prepare_start(self) -> None:
connection_config = ConnectionConfig.from_name(
self.normalized_name,
self.boot_info.trinity_config.ipc_dir,
)
await self.event_bus.start_serving(connection_config)
await self.event_bus.connect_to_endpoints(
ConnectionConfig.from_name(MAIN_EVENTBUS_ENDPOINT,
self.boot_info.trinity_config.ipc_dir))
# This makes the `main` process aware of this Endpoint which will then propagate the info
# so that every other Endpoint can connect directly to the plugin Endpoint
await self.event_bus.announce_endpoint()
await self.event_bus.broadcast(PluginStartedEvent(type(self)))
# Whenever new EventBus Endpoints come up the `main` process broadcasts this event
# and we connect to every Endpoint directly
asyncio.ensure_future(
self.event_bus.auto_connect_new_announced_endpoints())
self.do_start()
async def test_trio_duplicate_endpoint_connection_is_error(ipc_base_path):
config = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path)
async with TrioEndpoint.serve(config):
async with TrioEndpoint("client").run() as client:
await client.connect_to_endpoints(config)
assert client.is_connected_to(config.name)
with pytest.raises(ConnectionAttemptRejected):
await client.connect_to_endpoints(config)
def _prepare_start(self) -> None:
log_queue = self.context.boot_kwargs['log_queue']
level = self.context.boot_kwargs.get('log_level', logging.INFO)
setup_queue_logging(log_queue, level)
connection_config = ConnectionConfig.from_name(
self.normalized_name, self.context.trinity_config.ipc_dir)
self.event_bus.start_serving_nowait(connection_config)
self.event_bus.connect_to_endpoints_blocking(
ConnectionConfig.from_name(MAIN_EVENTBUS_ENDPOINT,
self.context.trinity_config.ipc_dir))
# This makes the `main` process aware of this Endpoint which will then propagate the info
# so that every other Endpoint can connect directly to the plugin Endpoint
self.event_bus.announce_endpoint()
self.event_bus.broadcast(PluginStartedEvent(type(self)))
# Whenever new EventBus Endpoints come up the `main` process broadcasts this event
# and we connect to every Endpoint directly
self.event_bus.auto_connect_new_announced_endpoints()
with self.context.trinity_config.process_id_file(self.normalized_name):
self.do_start()
def test_endpoint_broadcast_from_client_to_server(ipc_base_path, runner):
server_config = ConnectionConfig.from_name("server", base_path=ipc_base_path)
server = d.driver(d.serve_endpoint(server_config), d.wait_for(Event))
client = d.driver(
d.run_endpoint("client"),
d.connect_to_endpoints(server_config),
d.wait_any_then_broadcast(Event()),
)
runner(server, client)
async def worker(config: DriverProcessConfig) -> None:
conn_config = ConnectionConfig.from_name(DRIVER_ENDPOINT)
async with config.backend.Endpoint.serve(conn_config) as event_bus:
await event_bus.connect_to_endpoints(*config.connected_endpoints)
await event_bus.wait_until_all_remotes_subscribed_to(PerfMeasureEvent)
counter = itertools.count()
payload = b"\x00" * config.payload_bytes
while True:
await config.backend.sleep(config.throttle)
await event_bus.broadcast(
PerfMeasureEvent(payload, next(counter), time.time())
)
async def test_trio_server_endpoint_establishes_reverse_connection_to_client(
ipc_base_path):
unique_name = generate_unique_name()
config = ConnectionConfig.from_name(f"server-{unique_name}",
base_path=ipc_base_path)
async with TrioEndpoint.serve(config) as server:
async with TrioEndpoint(f"client-{unique_name}").run() as client:
await client.connect_to_endpoints(config)
assert client.is_connected_to(config.name)
with trio.fail_after(2):
await server.wait_until_connected_to(client.name)
async def launch_node_coro(args: Namespace,
trinity_config: TrinityConfig) -> None:
networking_connection_config = ConnectionConfig.from_name(
NETWORKING_EVENTBUS_ENDPOINT, trinity_config.ipc_dir)
async with TrinityEventBusEndpoint.serve(
networking_connection_config) as endpoint:
NodeClass = trinity_config.get_app_config(Eth1AppConfig).node_class
node = NodeClass(endpoint, trinity_config)
asyncio.ensure_future(endpoint.auto_connect_new_announced_endpoints())
await endpoint.connect_to_endpoints(
ConnectionConfig.from_name(MAIN_EVENTBUS_ENDPOINT,
trinity_config.ipc_dir), )
await endpoint.announce_endpoint()
# This is a second PluginManager instance governing plugins in a shared process.
plugin_manager = PluginManager(SharedProcessScope(endpoint),
get_plugins_for_eth1_client())
plugin_manager.prepare(args, trinity_config)
asyncio.ensure_future(
handle_networking_exit(node, plugin_manager, endpoint))
await node.run()
async def proc1_worker():
async with AsyncioEndpoint.serve(ConnectionConfig.from_name("e1")) as server:
server.subscribe(
SecondThingHappened,
lambda event: logging.info(
"Received via SUBSCRIBE API in proc1: %s", event.payload
),
)
await server.wait_until_any_endpoint_subscribed_to(FirstThingHappened)
while True:
logging.info("Hello from proc1")
await server.broadcast(FirstThingHappened("Hit from proc1"))
await asyncio.sleep(2)
async def worker(logger: logging.Logger,
config: ReportingProcessConfig) -> None:
conn_config = ConnectionConfig.from_name(REPORTER_ENDPOINT)
async with config.backend.Endpoint.serve(conn_config) as event_bus:
await event_bus.connect_to_endpoints(
ConnectionConfig.from_name(ROOT_ENDPOINT))
global_statistic = GlobalStatistic()
events = event_bus.stream(TotalRecordedEvent,
num_events=config.num_processes)
async for event in events:
global_statistic.add(event.total)
print_full_report(
logger,
config.backend,
config.num_processes,
config.num_events,
global_statistic,
)
await event_bus.broadcast(
ShutdownEvent(),
BroadcastConfig(filter_endpoint=ROOT_ENDPOINT))
async def test_base_wait_until_any_endpoint_subscriptions_changed():
config = ConnectionConfig.from_name(generate_unique_name())
async with AsyncioEndpoint.serve(config) as server:
async with AsyncioEndpoint("client").run() as client:
await client.connect_to_endpoints(config)
assert client.is_connected_to(config.name)
server.subscribe(SubscriptionEvent, lambda e: None)
assert not client.is_any_endpoint_subscribed_to(SubscriptionEvent)
await asyncio.wait_for(
client.wait_until_endpoint_subscriptions_change(), timeout=0.1
)
assert client.is_any_endpoint_subscribed_to(SubscriptionEvent)
def launch_node(args: Namespace, trinity_config: TrinityConfig) -> None:
with trinity_config.process_id_file('networking'):
endpoint = TrinityEventBusEndpoint()
NodeClass = trinity_config.get_app_config(Eth1AppConfig).node_class
node = NodeClass(endpoint, trinity_config)
# The `networking` process creates a process pool executor to offload cpu intensive
# tasks. We should revisit that when we move the sync in its own process
ensure_global_asyncio_executor()
loop = node.get_event_loop()
networking_connection_config = ConnectionConfig.from_name(
NETWORKING_EVENTBUS_ENDPOINT, trinity_config.ipc_dir)
endpoint.start_serving_nowait(
networking_connection_config,
loop,
)
endpoint.auto_connect_new_announced_endpoints()
endpoint.connect_to_endpoints_blocking(
ConnectionConfig.from_name(MAIN_EVENTBUS_ENDPOINT,
trinity_config.ipc_dir),
# Plugins that run within the networking process broadcast and receive on the
# the same endpoint
networking_connection_config,
)
endpoint.announce_endpoint()
# This is a second PluginManager instance governing plugins in a shared process.
plugin_manager = setup_plugins(SharedProcessScope(endpoint),
get_all_plugins())
plugin_manager.prepare(args, trinity_config)
asyncio.ensure_future(handle_networking_exit(node, plugin_manager,
endpoint),
loop=loop)
asyncio.ensure_future(node.run(), loop=loop)
loop.run_forever()
loop.close()
def test_broadcast_without_listeners_explicitly_allowed(ipc_base_path, runner):
server_config = ConnectionConfig.from_name("server", base_path=ipc_base_path)
server_done, client_done = d.checkpoint("done")
server = d.driver(d.serve_endpoint(server_config), server_done)
client = d.driver(
d.run_endpoint("client"),
d.connect_to_endpoints(server_config),
d.wait_until_connected_to("server"),
d.broadcast(Event(), BroadcastConfig(require_subscriber=False)),
client_done,
)
runner(server, client)
async def _run(self) -> None:
self._connection_config = ConnectionConfig.from_name(
MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir)
async with AsyncioEndpoint.serve(self._connection_config) as endpoint:
self._endpoint = endpoint
# start the background process that tracks and propagates available
# endpoints to the other connected endpoints
self.run_daemon_task(
self._track_and_propagate_available_endpoints())
# start the plugin manager
plugin_manager = PluginManager(endpoint, self._plugins)
plugin_manager.prepare(self._boot_info)
await self.cancellation()
def test_server_establishes_reverse_connection(ipc_base_path, runner):
server_config = ConnectionConfig.from_name("server",
base_path=ipc_base_path)
server_done, client_done = d.checkpoint("done")
server = d.driver(
d.serve_endpoint(server_config),
d.wait_until_connected_to("client"),
server_done,
)
client = d.driver(d.run_endpoint("client"),
d.connect_to_endpoints(server_config), client_done)
runner(server, client)
def test_broadcast_without_listeners_throws(ipc_base_path, runner):
server_config = ConnectionConfig.from_name("server", base_path=ipc_base_path)
server_done, client_done = d.checkpoint("done")
server = d.driver(d.serve_endpoint(server_config), server_done)
client = d.driver(
d.run_endpoint("client"),
d.connect_to_endpoints(server_config),
d.wait_until_connected_to("server"),
d.throws(d.broadcast(Event()), NoSubscribers),
client_done,
)
runner(server, client)
def __init__(self, position: Position, addr=None):
"""
:param position:
:param addr: requested address for the node
"""
super().__init__()
self.start_time = 0
self.network = None
self.position = position
self.addr = self._get_free_address(addr)
self.bus_config = ConnectionConfig.from_name("network")
self.kill_switch: Value = Value("i", 0)
self.nodes_history = {}
async def proc2_worker():
config = ConnectionConfig.from_name("e1")
async with AsyncioEndpoint("e2").run() as client:
await client.connect_to_endpoints(config)
asyncio.ensure_future(display_proc1_events(client))
client.subscribe(
FirstThingHappened,
lambda event: logging.info(
"Received via SUBSCRIBE API in proc2: %s", event.payload
),
)
await client.wait_until_any_endpoint_subscribed_to(SecondThingHappened)
while True:
logging.info("Hello from proc2")
await client.broadcast(SecondThingHappened("Hit from proc2 "))
await asyncio.sleep(2)
def test_duplicate_connection_throws_exception(ipc_base_path, runner):
server_config = ConnectionConfig.from_name("server",
base_path=ipc_base_path)
server_done, client_done = d.checkpoint("done")
server = d.driver(d.serve_endpoint(server_config), server_done)
client = d.driver(
d.run_endpoint("client"),
d.connect_to_endpoints(server_config),
d.wait_until_connected_to("server"),
d.throws(d.connect_to_endpoints(server_config),
ConnectionAttemptRejected),
client_done,
)
runner(server, client)
async def trinity_boot_coro(kill_trinity, main_endpoint, trinity_config, # type: ignore
plugin_manager, args, extra_kwargs) -> None:
# We postpone EventBus connection until here because we don't want one in cases where
# a plugin just redefines the `trinity` command such as `trinity fix-unclean-shutdown`
main_connection_config = ConnectionConfig.from_name(
MAIN_EVENTBUS_ENDPOINT,
trinity_config.ipc_dir
)
await main_endpoint.start()
await main_endpoint.start_server(main_connection_config.path)
main_endpoint.track_and_propagate_available_endpoints()
main_endpoint.subscribe(
ShutdownRequest,
lambda ev: kill_trinity(ev.reason)
)
plugin_manager.prepare(args, trinity_config, extra_kwargs)
def test_request_response(runner, ipc_base_path):
server_config = ConnectionConfig.from_name("server", base_path=ipc_base_path)
received = multiprocessing.Event()
server = d.driver(
d.serve_endpoint(server_config),
d.serve_request(Request, get_response=lambda endpoint, event: Response()),
)
client = d.driver(
d.run_endpoint("client"),
d.connect_to_endpoints(server_config),
d.wait_until_any_endpoint_subscribed_to(Request),
d.request(Request(), on_response=lambda endpoint, event: received.set()),
)
runner(server, client)
assert received.is_set()
