async def launch_node_coro(args: Namespace,
trinity_config: TrinityConfig) -> None:
endpoint = TrinityEventBusEndpoint()
NodeClass = trinity_config.get_app_config(Eth1AppConfig).node_class
node = NodeClass(endpoint, trinity_config)
networking_connection_config = ConnectionConfig.from_name(
NETWORKING_EVENTBUS_ENDPOINT, trinity_config.ipc_dir)
await endpoint.start_serving(networking_connection_config)
endpoint.auto_connect_new_announced_endpoints()
await endpoint.connect_to_endpoints(
ConnectionConfig.from_name(MAIN_EVENTBUS_ENDPOINT,
trinity_config.ipc_dir),
# Plugins that run within the networking process broadcast and receive on the
# the same endpoint
networking_connection_config,
)
await endpoint.announce_endpoint()
# This is a second PluginManager instance governing plugins in a shared process.
plugin_manager = PluginManager(SharedProcessScope(endpoint),
get_all_plugins())
plugin_manager.prepare(args, trinity_config)
asyncio.ensure_future(
handle_networking_exit(node, plugin_manager, endpoint))
asyncio.ensure_future(node.run())
async def _run_eventbus_for_component(
component: Union['TrioIsolatedComponent', 'AsyncioIsolatedComponent'],
connect_to_endpoints: Tuple[str, ...],
) -> AsyncIterator[Union[TrioEndpoint, AsyncioEndpoint]]:
from trinity.extensibility.trio import TrioIsolatedComponent
from trinity.extensibility.asyncio import AsyncioIsolatedComponent
if isinstance(component, TrioIsolatedComponent):
endpoint_type: Union[Type[TrioEndpoint],
Type[AsyncioEndpoint]] = TrioEndpoint
elif isinstance(component, AsyncioIsolatedComponent):
endpoint_type = AsyncioEndpoint
else:
raise ValueError("Unknown component type: %s", type(component))
trinity_config = component._boot_info.trinity_config
conn_config = ConnectionConfig.from_name(component.get_endpoint_name(),
trinity_config.ipc_dir)
async with endpoint_type.serve(conn_config) as event_bus:
for endpoint in connect_to_endpoints:
path = pathlib.Path(endpoint)
if not path.is_socket():
raise ValueError("Invalid IPC path: {path}")
connection_config = ConnectionConfig(name=path.stem, path=path)
logger.info("Attempting to connect to eventbus endpoint at %s",
connection_config)
await event_bus.connect_to_endpoints(connection_config)
yield event_bus
async def proc2_worker():
endpoint = Endpoint()
await endpoint.start_serving(ConnectionConfig.from_name('e2'))
await endpoint.connect_to_endpoints(ConnectionConfig.from_name('e1'), )
for i in range(3):
print("Requesting")
result = await endpoint.request(GetSomethingRequest())
print(f"Got answer: {result.payload}")
async def pair_of_endpoints(event_loop, ipc_base_path):
config_1 = ConnectionConfig.from_name(generate_unique_name(),
base_path=ipc_base_path)
config_2 = ConnectionConfig.from_name(generate_unique_name(),
base_path=ipc_base_path)
async with AsyncioEndpoint.serve(config_1) as endpoint1:
async with AsyncioEndpoint.serve(config_2) as endpoint2:
await endpoint1.connect_to_endpoints(config_2)
await endpoint2.connect_to_endpoints(config_1)
yield endpoint1, endpoint2
async def run_proc1():
endpoint = Endpoint()
await endpoint.start_serving(ConnectionConfig.from_name('e1'))
await endpoint.connect_to_endpoints(ConnectionConfig.from_name('e2'), )
print("subscribing")
# Listen for `GetSomethingRequest`'s
endpoint.subscribe(
GetSomethingRequest,
lambda event:
# Send a response back to *only* who made that request
endpoint.broadcast_nowait(DeliverSomethingResponse("Yay"),
event.broadcast_config()))
async def client_with_three_connections(ipc_base_path):
config_a = ConnectionConfig.from_name("server-a", base_path=ipc_base_path)
config_b = ConnectionConfig.from_name("server-b", base_path=ipc_base_path)
config_c = ConnectionConfig.from_name("server-c", base_path=ipc_base_path)
async with AsyncioEndpoint.serve(config_a) as server_a:
async with AsyncioEndpoint.serve(config_b) as server_b:
async with AsyncioEndpoint.serve(config_c) as server_c:
async with AsyncioEndpoint("client").run() as client:
await client.connect_to_endpoints(config_a)
await client.connect_to_endpoints(config_b)
await client.connect_to_endpoints(config_c)
yield client, server_a, server_b, server_c
async def proc2_worker():
endpoint = Endpoint()
await endpoint.start_serving(ConnectionConfig.from_name('e2'))
await endpoint.connect_to_endpoints(ConnectionConfig.from_name('e1'))
asyncio.ensure_future(display_proc1_events(endpoint))
endpoint.subscribe(
FirstThingHappened, lambda event: print(
"Received via SUBSCRIBE API in proc2:", event.payload))
while True:
print("Hello from proc2")
if is_nth_second(2):
await endpoint.broadcast(
SecondThingHappened("Hit from proc2 ({})".format(time.time())))
await asyncio.sleep(1)
async def worker(cls, name: str, config: ConsumerConfig) -> None:
conn_config = ConnectionConfig.from_name(name)
async with config.backend.Endpoint.serve(conn_config) as event_bus:
await event_bus.connect_to_endpoints(
ConnectionConfig.from_name(REPORTER_ENDPOINT))
await event_bus.wait_until_connected_to(DRIVER_ENDPOINT)
stats = await cls.do_consumer(event_bus, config)
await event_bus.wait_until_endpoint_subscribed_to(
REPORTER_ENDPOINT, TotalRecordedEvent)
await event_bus.broadcast(
TotalRecordedEvent(stats.crunch(event_bus.name)),
BroadcastConfig(filter_endpoint=REPORTER_ENDPOINT),
)
async def _run(self) -> None:
async with AsyncioEndpoint.serve(self._connection_config) as endpoint:
self._endpoint = endpoint
# signal that the endpoint is now available
self._endpoint_available.set()
# run background task that automatically connects to newly announced endpoints
self.run_daemon_task(self._auto_connect_new_announced_endpoints())
# connect to the *main* endpoint which communicates information
# about other endpoints that come online.
main_endpoint_config = ConnectionConfig.from_name(
MAIN_EVENTBUS_ENDPOINT,
self._trinity_config.ipc_dir,
)
await endpoint.connect_to_endpoints(main_endpoint_config)
# announce ourself to the event bus
await endpoint.wait_until_endpoint_subscribed_to(
MAIN_EVENTBUS_ENDPOINT,
EventBusConnected,
)
await endpoint.broadcast(
EventBusConnected(self._connection_config),
BroadcastConfig(filter_endpoint=MAIN_EVENTBUS_ENDPOINT))
# run until the endpoint exits
await self.cancellation()
def create_consumer_endpoint_configs(
num_processes: int
) -> Tuple[ConnectionConfig, ...]:
return tuple(
ConnectionConfig.from_name(create_consumer_endpoint_name(i))
for i in range(num_processes)
)
async def make_networking_event_bus():
# Tests run concurrently, therefore we need unique IPC paths
ipc_path = Path(f"networking-{uuid.uuid4()}.ipc")
networking_connection_config = ConnectionConfig(
name=NETWORKING_EVENTBUS_ENDPOINT, path=ipc_path)
async with AsyncioEndpoint.serve(networking_connection_config) as endpoint:
yield endpoint
async def run_proc1():
config = ConnectionConfig.from_name("e1")
async with AsyncioEndpoint.serve(config) as server:
async for event in server.stream(GetSomethingRequest, num_events=3):
await server.broadcast(
DeliverSomethingResponse("Yay"), event.broadcast_config()
)
async def run(args: argparse.Namespace, backend: BaseBackend):
consumer_endpoint_configs = create_consumer_endpoint_configs(args.num_processes)
(
config.path.unlink()
for config in consumer_endpoint_configs
+ tuple(
ConnectionConfig.from_name(name)
for name in (ROOT_ENDPOINT, REPORTER_ENDPOINT, DRIVER_ENDPOINT)
)
)
root_config = ConnectionConfig.from_name(ROOT_ENDPOINT)
async with backend.Endpoint.serve(root_config) as root:
# The reporter process is collecting statistical events from all consumer processes
# For some reason, doing this work in the main process didn't end so well which is
# why it was moved into a dedicated process. Notice that this will slightly skew results
# as the reporter process will also receive events which we don't account for
reporting_config = ReportingProcessConfig(
num_events=args.num_events,
num_processes=args.num_processes,
throttle=args.throttle,
payload_bytes=args.payload_bytes,
backend=backend,
)
reporter = ReportingProcess(reporting_config)
reporter.start()
for n in range(args.num_processes):
consumer_process = ConsumerProcess(
create_consumer_endpoint_name(n), args.num_events, backend=backend
)
consumer_process.start()
# In this benchmark, this is the only process that is flooding events
driver_config = DriverProcessConfig(
connected_endpoints=consumer_endpoint_configs,
num_events=args.num_events,
throttle=args.throttle,
payload_bytes=args.payload_bytes,
backend=backend,
)
driver = DriverProcess(driver_config)
driver.start()
await root.wait_for(ShutdownEvent)
driver.stop()
async def test_wait_until_connected_to(endpoint):
config = ConnectionConfig.from_name(generate_unique_name())
async with AsyncioEndpoint.serve(config):
asyncio.ensure_future(endpoint.connect_to_endpoints(config))
assert not endpoint.is_connected_to(config.name)
await endpoint.wait_until_connected_to(config.name)
assert endpoint.is_connected_to(config.name)
async def test_connecting_to_other_trio_endpoint(ipc_base_path):
config = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path)
async with TrioEndpoint.serve(config):
async with TrioEndpoint("client").run() as client:
await client.connect_to_endpoints(config)
assert client.is_connected_to(config.name)
async def _prepare_start(self) -> None:
connection_config = ConnectionConfig.from_name(
self.normalized_name, self.boot_info.trinity_config.ipc_dir)
await self.event_bus.start_serving(connection_config)
await self.event_bus.connect_to_endpoints(
ConnectionConfig.from_name(MAIN_EVENTBUS_ENDPOINT,
self.boot_info.trinity_config.ipc_dir))
# This makes the `main` process aware of this Endpoint which will then propagate the info
# so that every other Endpoint can connect directly to the plugin Endpoint
await self.event_bus.announce_endpoint()
await self.event_bus.broadcast(PluginStartedEvent(type(self)))
# Whenever new EventBus Endpoints come up the `main` process broadcasts this event
# and we connect to every Endpoint directly
self.event_bus.auto_connect_new_announced_endpoints()
self.do_start()
async def test_endpoint_serve(ipc_base_path):
config = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path)
async with AsyncioEndpoint.serve(config) as endpoint:
assert endpoint.is_running is True
assert endpoint.is_serving is True
assert endpoint.is_running is False
assert endpoint.is_serving is False
async def test_rejects_duplicates_when_connecting():
own = ConnectionConfig.from_name(generate_unique_name())
async with AsyncioEndpoint.serve(own) as endpoint:
await endpoint.connect_to_endpoint(own)
assert endpoint.is_connected_to(own.name)
with pytest.raises(ConnectionAttemptRejected):
await endpoint.connect_to_endpoint(own)
async def monitoring(normalized_name: str,
trinity_config: TrinityConfig) -> None:
event_bus = TrinityEventBusEndpoint("monitoring_ui")
connection_config = ConnectionConfig.from_name(
normalized_name,
trinity_config.ipc_dir,
)
await event_bus.start()
await event_bus.start_server(connection_config.path)
await event_bus.connect_to_endpoints(
ConnectionConfig.from_name(MAIN_EVENTBUS_ENDPOINT,
trinity_config.ipc_dir))
await event_bus.announce_endpoint()
await event_bus.broadcast(PluginStartedEvent(type(MonitoringPlugin)))
asyncio.ensure_future(event_bus.auto_connect_new_announced_endpoints())
event_bus.subscribe(NewBlockHashesEvent,
lambda event: logging.info(event.msg))
def __init__(
self,
trinity_config: TrinityConfig,
endpoint_name: str,
) -> None:
self._trinity_config = trinity_config
self._endpoint_available = self.EventType()
self._connection_config = ConnectionConfig.from_name(
endpoint_name, self._trinity_config.ipc_dir)
async def endpoint(event_loop, ipc_base_path):
config = ConnectionConfig.from_name(generate_unique_name(),
base_path=ipc_base_path)
async with AsyncioEndpoint.serve(config) as endpoint:
# We need to connect to our own Endpoint if we care about receiving
# the events we broadcast. Many tests use the same Endpoint for
# broadcasting and receiving which is a valid use case so we hook it up
await endpoint.connect_to_endpoint(config)
yield endpoint
async def announce_endpoint(self) -> None:
"""
Announce this endpoint to the :class:`~trinity.endpoint.TrinityMainEventBusEndpoint` so
that it will be further propagated to all other endpoints, allowing them to connect to us.
"""
await self.broadcast(
EventBusConnected(
ConnectionConfig(name=self.name, path=self.ipc_path)),
BroadcastConfig(filter_endpoint=MAIN_EVENTBUS_ENDPOINT))
async def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('-bootnode',
type=str,
help="The enode to use as bootnode")
parser.add_argument('-networkid',
type=int,
choices=[ROPSTEN_NETWORK_ID, MAINNET_NETWORK_ID],
default=ROPSTEN_NETWORK_ID,
help="1 for mainnet, 3 for testnet")
parser.add_argument('-l', type=str, help="Log level", default="info")
args = parser.parse_args()
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s: %(message)s',
datefmt='%H:%M:%S')
if args.l == "debug2": # noqa: E741
log_level = DEBUG2_LEVEL_NUM
else:
log_level = getattr(logging, args.l.upper())
logging.getLogger('p2p').setLevel(log_level)
network_cfg = PRECONFIGURED_NETWORKS[args.networkid]
# Listen on a port other than 30303 so that we can test against a local geth instance
# running on that port.
listen_port = 30304
# Use a hard-coded privkey so that our enode is always the same.
privkey = keys.PrivateKey(
b'~\x054{4\r\xd64\x0f\x98\x1e\x85;\xcc\x08\x1eQ\x10t\x16\xc0\xb0\x7f)=\xc4\x1b\xb7/\x8b&\x83'
) # noqa: E501
addr = kademlia.Address('127.0.0.1', listen_port, listen_port)
if args.bootnode:
bootstrap_nodes = tuple([kademlia.Node.from_uri(args.bootnode)])
else:
bootstrap_nodes = tuple(
kademlia.Node.from_uri(enode) for enode in network_cfg.bootnodes)
ipc_path = Path(f"networking-{uuid.uuid4()}.ipc")
networking_connection_config = ConnectionConfig(
name=NETWORKING_EVENTBUS_ENDPOINT, path=ipc_path)
headerdb = TrioHeaderDB(AtomicDB(MemoryDB()))
headerdb.persist_header(network_cfg.genesis_header)
vm_config = network_cfg.vm_configuration
enr_field_providers = (functools.partial(generate_eth_cap_enr_field,
vm_config, headerdb), )
socket = trio.socket.socket(family=trio.socket.AF_INET,
type=trio.socket.SOCK_DGRAM)
await socket.bind(('0.0.0.0', listen_port))
async with TrioEndpoint.serve(networking_connection_config) as endpoint:
service = DiscoveryService(privkey, addr, bootstrap_nodes, endpoint,
socket, enr_field_providers)
service.logger.info("Enode: %s", service.this_node.uri())
async with background_trio_service(service):
await service.manager.wait_finished()
async def worker(backend: BaseBackend, name: str, num_events: int) -> None:
config = ConnectionConfig.from_name(name)
async with backend.Endpoint.serve(config) as event_bus:
await event_bus.connect_to_endpoints(
ConnectionConfig.from_name(REPORTER_ENDPOINT)
)
await event_bus.wait_until_all_remotes_subscribed_to(TotalRecordedEvent)
stats = LocalStatistic()
events = event_bus.stream(PerfMeasureEvent, num_events=num_events)
async for event in events:
stats.add(
RawMeasureEntry(sent_at=event.sent_at, received_at=time.time())
)
await event_bus.broadcast(
TotalRecordedEvent(stats.crunch(event_bus.name)),
BroadcastConfig(filter_endpoint=REPORTER_ENDPOINT),
)
def test_request_without_subscriber_throws(runner, ipc_base_path):
server_config = ConnectionConfig.from_name("server", base_path=ipc_base_path)
server = d.driver(d.serve_endpoint(server_config))
client = d.driver(
d.run_endpoint("client"), d.throws(d.request(Request()), NoSubscribers)
)
runner(server, client)
async def proc2_worker():
config = ConnectionConfig.from_name("e1")
async with AsyncioEndpoint("e2").run() as client:
await client.connect_to_endpoints(config)
await client.wait_until_any_endpoint_subscribed_to(GetSomethingRequest)
for i in range(3):
print("Requesting")
result = await client.request(GetSomethingRequest())
print(f"Got answer: {result.payload}")
async def test_base_wait_until_connections_changed():
config = ConnectionConfig.from_name(generate_unique_name())
async with AsyncioEndpoint.serve(config):
async with AsyncioEndpoint("client").run() as client:
asyncio.ensure_future(client.connect_to_endpoints(config))
assert not client.is_connected_to(config.name)
await asyncio.wait_for(client.wait_until_connections_change(),
timeout=0.1)
assert client.is_connected_to(config.name)
def __init__(self,
trinity_config: TrinityConfig,
endpoint_name: str,
cancel_token: CancelToken = None,
loop: asyncio.AbstractEventLoop = None) -> None:
self._trinity_config = trinity_config
self._endpoint_available = asyncio.Event()
self._connection_config = ConnectionConfig.from_name(
endpoint_name, self._trinity_config.ipc_dir)
super().__init__(cancel_token, loop)
async def run_network(self):
async with AsyncioEndpoint.serve(
ConnectionConfig.from_name("network")) as self.network_bus:
self.logger.info("Network started")
self.network_bus.subscribe(GenericMessageOutgoingEvent,
self.send_to_all_nodes)
self.logger.info("Network subscribed to all messages")
while not self.kill_switch.value:
await asyncio.sleep(0)
self.logger.info("Network turned off")
async def run(self) -> None:
connection_config = ConnectionConfig.from_name(
MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir)
async with AsyncioEndpoint.serve(connection_config) as endpoint:
self._endpoint = endpoint
# start the background process that tracks and propagates available
# endpoints to the other connected endpoints
self.manager.run_daemon_task(
self._track_and_propagate_available_endpoints)
await endpoint.wait_until_any_endpoint_subscribed_to(
EventBusConnected)
# signal the endpoint is up and running and available
self._endpoint_available.set()
# instantiate all of the components
all_components = tuple(
component_cls(self._boot_info)
for component_cls in self._component_types)
# filter out any components that should not be enabled.
enabled_components = tuple(component
for component in all_components
if component.is_enabled)
# a little bit of extra try/finally structure here to produce good
# logging messages about the component lifecycle.
from p2p.asyncio_utils import create_task, wait_first
try:
self.logger.info(
"Starting components: %s",
'/'.join(component.name
for component in enabled_components),
)
tasks: List[asyncio.Task[Any]] = []
for component in enabled_components:
tasks.append(
create_task(
component.run_in_process(),
f'IsolatedComponent/{component.name}/run_in_process'
))
tasks.append(
asyncio.create_task(self._trigger_component_exit.wait()))
self.logger.info("Components started")
try:
# The timeout is long as our component tasks can do a lot of stuff during
# their cleanup.
await wait_first(tasks, max_wait_after_cancellation=10)
finally:
self.logger.info("Stopping components")
finally:
self.logger.info("Components stopped.")
self.manager.cancel()
