async def test_endpoint_run():
endpoint = AsyncioEndpoint("test-run")
assert endpoint.is_running is False
async with endpoint.run():
assert endpoint.is_running is True
assert endpoint.is_running is False
async def test_base_wait_until_connections_changed():
config = ConnectionConfig.from_name(generate_unique_name())
async with AsyncioEndpoint.serve(config):
async with AsyncioEndpoint("client").run() as client:
asyncio.ensure_future(client.connect_to_endpoints(config))
assert not client.is_connected_to(config.name)
await asyncio.wait_for(client.wait_until_connections_change(),
timeout=0.1)
assert client.is_connected_to(config.name)
async def pair_of_endpoints(event_loop, ipc_base_path):
config_1 = ConnectionConfig.from_name(generate_unique_name(),
base_path=ipc_base_path)
config_2 = ConnectionConfig.from_name(generate_unique_name(),
base_path=ipc_base_path)
async with AsyncioEndpoint.serve(config_1) as endpoint1:
async with AsyncioEndpoint.serve(config_2) as endpoint2:
await endpoint1.connect_to_endpoints(config_2)
await endpoint2.connect_to_endpoints(config_1)
yield endpoint1, endpoint2
async def test_endpoint_run_with_error():
endpoint = AsyncioEndpoint("test-run")
assert endpoint.is_running is False
with pytest.raises(Exception, match="break out of run"):
async with endpoint.run():
assert endpoint.is_running is True
raise Exception("break out of run")
assert endpoint.is_running is False
async def test_base_wait_until_any_endpoint_subscriptions_changed():
config = ConnectionConfig.from_name(generate_unique_name())
async with AsyncioEndpoint.serve(config) as server:
async with AsyncioEndpoint("client").run() as client:
await client.connect_to_endpoints(config)
assert client.is_connected_to(config.name)
server.subscribe(SubscriptionEvent, lambda e: None)
assert not client.is_any_endpoint_subscribed_to(SubscriptionEvent)
await asyncio.wait_for(
client.wait_until_endpoint_subscriptions_change(), timeout=0.1
)
assert client.is_any_endpoint_subscribed_to(SubscriptionEvent)
async def client_with_three_connections(ipc_base_path):
config_a = ConnectionConfig.from_name("server-a", base_path=ipc_base_path)
config_b = ConnectionConfig.from_name("server-b", base_path=ipc_base_path)
config_c = ConnectionConfig.from_name("server-c", base_path=ipc_base_path)
async with AsyncioEndpoint.serve(config_a) as server_a:
async with AsyncioEndpoint.serve(config_b) as server_b:
async with AsyncioEndpoint.serve(config_c) as server_c:
async with AsyncioEndpoint("client").run() as client:
await client.connect_to_endpoints(config_a)
await client.connect_to_endpoints(config_b)
await client.connect_to_endpoints(config_c)
yield client, server_a, server_b, server_c
async def _run(self) -> None:
async with AsyncioEndpoint.serve(self._connection_config) as endpoint:
self._endpoint = endpoint
# signal that the endpoint is now available
self._endpoint_available.set()
# run background task that automatically connects to newly announced endpoints
self.run_daemon_task(self._auto_connect_new_announced_endpoints())
# connect to the *main* endpoint which communicates information
# about other endpoints that come online.
main_endpoint_config = ConnectionConfig.from_name(
MAIN_EVENTBUS_ENDPOINT,
self._trinity_config.ipc_dir,
)
await endpoint.connect_to_endpoints(main_endpoint_config)
# announce ourself to the event bus
await endpoint.wait_until_endpoint_subscribed_to(
MAIN_EVENTBUS_ENDPOINT,
EventBusConnected,
)
await endpoint.broadcast(
EventBusConnected(self._connection_config),
BroadcastConfig(filter_endpoint=MAIN_EVENTBUS_ENDPOINT))
# run until the endpoint exits
await self.cancellation()
async def run_proc1():
config = ConnectionConfig.from_name("e1")
async with AsyncioEndpoint.serve(config) as server:
async for event in server.stream(GetSomethingRequest, num_events=3):
await server.broadcast(
DeliverSomethingResponse("Yay"), event.broadcast_config()
)
async def make_networking_event_bus():
# Tests run concurrently, therefore we need unique IPC paths
ipc_path = Path(f"networking-{uuid.uuid4()}.ipc")
networking_connection_config = ConnectionConfig(
name=NETWORKING_EVENTBUS_ENDPOINT, path=ipc_path)
async with AsyncioEndpoint.serve(networking_connection_config) as endpoint:
yield endpoint
async def test_endpoint_serve(ipc_base_path):
config = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path)
async with AsyncioEndpoint.serve(config) as endpoint:
assert endpoint.is_running is True
assert endpoint.is_serving is True
assert endpoint.is_running is False
assert endpoint.is_serving is False
async def test_rejects_duplicates_when_connecting():
own = ConnectionConfig.from_name(generate_unique_name())
async with AsyncioEndpoint.serve(own) as endpoint:
await endpoint.connect_to_endpoint(own)
assert endpoint.is_connected_to(own.name)
with pytest.raises(ConnectionAttemptRejected):
await endpoint.connect_to_endpoint(own)
async def test_wait_until_connected_to(endpoint):
config = ConnectionConfig.from_name(generate_unique_name())
async with AsyncioEndpoint.serve(config):
asyncio.ensure_future(endpoint.connect_to_endpoints(config))
assert not endpoint.is_connected_to(config.name)
await endpoint.wait_until_connected_to(config.name)
assert endpoint.is_connected_to(config.name)
async def endpoint(event_loop, ipc_base_path):
config = ConnectionConfig.from_name(generate_unique_name(),
base_path=ipc_base_path)
async with AsyncioEndpoint.serve(config) as endpoint:
# We need to connect to our own Endpoint if we care about receiving
# the events we broadcast. Many tests use the same Endpoint for
# broadcasting and receiving which is a valid use case so we hook it up
await endpoint.connect_to_endpoint(config)
yield endpoint
async def run_network(self):
async with AsyncioEndpoint.serve(
ConnectionConfig.from_name("network")) as self.network_bus:
self.logger.info("Network started")
self.network_bus.subscribe(GenericMessageOutgoingEvent,
self.send_to_all_nodes)
self.logger.info("Network subscribed to all messages")
while not self.kill_switch.value:
await asyncio.sleep(0)
self.logger.info("Network turned off")
async def proc2_worker():
config = ConnectionConfig.from_name("e1")
async with AsyncioEndpoint("e2").run() as client:
await client.connect_to_endpoints(config)
await client.wait_until_any_endpoint_subscribed_to(GetSomethingRequest)
for i in range(3):
print("Requesting")
result = await client.request(GetSomethingRequest())
print(f"Got answer: {result.payload}")
async def run(self) -> None:
connection_config = ConnectionConfig.from_name(
MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir)
async with AsyncioEndpoint.serve(connection_config) as endpoint:
self._endpoint = endpoint
# start the background process that tracks and propagates available
# endpoints to the other connected endpoints
self.manager.run_daemon_task(
self._track_and_propagate_available_endpoints)
await endpoint.wait_until_any_endpoint_subscribed_to(
EventBusConnected)
# signal the endpoint is up and running and available
self._endpoint_available.set()
# instantiate all of the components
all_components = tuple(
component_cls(self._boot_info)
for component_cls in self._component_types)
# filter out any components that should not be enabled.
enabled_components = tuple(component
for component in all_components
if component.is_enabled)
# a little bit of extra try/finally structure here to produce good
# logging messages about the component lifecycle.
from p2p.asyncio_utils import create_task, wait_first
try:
self.logger.info(
"Starting components: %s",
'/'.join(component.name
for component in enabled_components),
)
tasks: List[asyncio.Task[Any]] = []
for component in enabled_components:
tasks.append(
create_task(
component.run_in_process(),
f'IsolatedComponent/{component.name}/run_in_process'
))
tasks.append(
asyncio.create_task(self._trigger_component_exit.wait()))
self.logger.info("Components started")
try:
# The timeout is long as our component tasks can do a lot of stuff during
# their cleanup.
await wait_first(tasks, max_wait_after_cancellation=10)
finally:
self.logger.info("Stopping components")
finally:
self.logger.info("Components stopped.")
self.manager.cancel()
async def run(self) -> None:
connection_config = ConnectionConfig.from_name(
MAIN_EVENTBUS_ENDPOINT,
self._boot_info.trinity_config.ipc_dir
)
async with AsyncioEndpoint.serve(connection_config) as endpoint:
self._endpoint = endpoint
# start the background process that tracks and propagates available
# endpoints to the other connected endpoints
self.manager.run_daemon_task(self._track_and_propagate_available_endpoints)
self.manager.run_daemon_task(self._handle_shutdown_request)
await endpoint.wait_until_any_endpoint_subscribed_to(ShutdownRequest)
await endpoint.wait_until_any_endpoint_subscribed_to(EventBusConnected)
# signal the endpoint is up and running and available
self._endpoint_available.set()
# instantiate all of the components
all_components = tuple(
component_cls(self._boot_info)
for component_cls
in self._component_types
)
# filter out any components that should not be enabled.
enabled_components = tuple(
component
for component in all_components
if component.is_enabled
)
# a little bit of extra try/finally structure here to produce good
# logging messages about the component lifecycle.
try:
async with AsyncExitStack() as stack:
self.logger.info(
"Starting components: %s",
'/'.join(component.name for component in enabled_components),
)
# Concurrently start the components.
await asyncio.gather(*(
stack.enter_async_context(run_component(component))
for component in enabled_components
))
self.logger.info("Components started")
try:
await self._trigger_component_exit.wait()
finally:
self.logger.info("Stopping components")
finally:
self.logger.info("Components stopped.")
self.manager.cancel()
async def run(self) -> None:
connection_config = ConnectionConfig.from_name(
MAIN_EVENTBUS_ENDPOINT,
self._boot_info.trinity_config.ipc_dir
)
async with AsyncioEndpoint.serve(connection_config) as endpoint:
self._endpoint = endpoint
# start the background process that tracks and propagates available
# endpoints to the other connected endpoints
self.manager.run_daemon_task(self._track_and_propagate_available_endpoints)
await endpoint.wait_until_any_endpoint_subscribed_to(EventBusConnected)
# signal the endpoint is up and running and available
self._endpoint_available.set()
# instantiate all of the components
all_components = tuple(
component_cls(self._boot_info)
for component_cls
in self._component_types
)
# filter out any components that should not be enabled.
enabled_components = tuple(
component
for component in all_components
if component.is_enabled
)
# a little bit of extra try/finally structure here to produce good
# logging messages about the component lifecycle.
from p2p.logic import wait_first
try:
self.logger.info(
"Starting components: %s",
'/'.join(component.name for component in enabled_components),
)
context_managers = [component.run() for component in enabled_components]
async with AsyncContextGroup(context_managers) as futs:
# AsyncContextGroup() yields a Sequence[Any], so we cast to a list of Futures
# here to ensure mypy can come to our aid if we forget the create_task() when
# adding new entries to the list of Futures we want to wait for.
futures = cast(List["asyncio.Future[None]"], list(futs))
futures.append(asyncio.create_task(self._trigger_component_exit.wait()))
self.logger.info("Components started")
try:
await wait_first(futures)
finally:
self.logger.info("Stopping components")
finally:
self.logger.info("Components stopped.")
self.manager.cancel()
async def test_endpoint_serve_with_error(ipc_base_path):
config = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path)
with pytest.raises(Exception, match="break out of serve"):
async with AsyncioEndpoint.serve(config) as endpoint:
assert endpoint.is_running is True
assert endpoint.is_serving is True
raise Exception("break out of serve")
assert endpoint.is_running is False
assert endpoint.is_serving is False
async def run(self, nodes_history):
self.nodes_history = nodes_history
self.nodes_history[self.addr] = []
async with AsyncioEndpoint(
f"node_{self.addr}").run() as self.network_bus:
await self.connect_to_network()
self.start_time = time.time()
while not self.kill_switch.value:
# self.logger.debug(f"[{self}] Kill_switch state: {self.kill_switch.value}")
await self.main_message_tick()
self.logger.info(f"Node_{self.addr} killed")
self.network_bus = None
async def proc1_worker():
async with AsyncioEndpoint.serve(ConnectionConfig.from_name("e1")) as server:
server.subscribe(
SecondThingHappened,
lambda event: logging.info(
"Received via SUBSCRIBE API in proc1: %s", event.payload
),
)
await server.wait_until_any_endpoint_subscribed_to(FirstThingHappened)
while True:
logging.info("Hello from proc1")
await server.broadcast(FirstThingHappened("Hit from proc1"))
await asyncio.sleep(2)
async def _run(self) -> None:
self._connection_config = ConnectionConfig.from_name(
MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir)
async with AsyncioEndpoint.serve(self._connection_config) as endpoint:
self._endpoint = endpoint
# start the background process that tracks and propagates available
# endpoints to the other connected endpoints
self.run_daemon_task(
self._track_and_propagate_available_endpoints())
# start the plugin manager
plugin_manager = PluginManager(endpoint, self._plugins)
plugin_manager.prepare(self._boot_info)
await self.cancellation()
async def proc2_worker():
config = ConnectionConfig.from_name("e1")
async with AsyncioEndpoint("e2").run() as client:
await client.connect_to_endpoints(config)
asyncio.ensure_future(display_proc1_events(client))
client.subscribe(
FirstThingHappened,
lambda event: logging.info(
"Received via SUBSCRIBE API in proc2: %s", event.payload
),
)
await client.wait_until_any_endpoint_subscribed_to(SecondThingHappened)
while True:
logging.info("Hello from proc2")
await client.broadcast(SecondThingHappened("Hit from proc2 "))
await asyncio.sleep(2)
async def test_beam_syncer(request,
event_loop,
event_bus,
chaindb_fresh,
chaindb_churner,
beam_to_block,
checkpoint=None):
client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db)
server_context = ChainContextFactory(headerdb__db=chaindb_churner.db)
peer_pair = ETHPeerPairFactory(
alice_peer_context=client_context,
bob_peer_context=server_context,
event_bus=event_bus,
)
async with peer_pair as (client_peer, server_peer):
# Need a name that will be unique per xdist-process, otherwise
# lahja IPC endpoints in each process will clobber each other
unique_process_name = uuid.uuid4()
# manually add endpoint for beam vm to make requests
pausing_config = ConnectionConfig.from_name(
f"PausingEndpoint-{unique_process_name}")
# manually add endpoint for trie data gatherer to serve requests
gatherer_config = ConnectionConfig.from_name(
f"GathererEndpoint-{unique_process_name}")
client_peer_pool = MockPeerPoolWithConnectedPeers([client_peer])
server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer],
event_bus=event_bus)
async with run_peer_pool_event_server(
event_bus, server_peer_pool,
handler_type=ETHPeerPoolEventServer), run_request_server(
event_bus, AsyncChainDB(chaindb_churner.db)
), AsyncioEndpoint.serve(
pausing_config) as pausing_endpoint, AsyncioEndpoint.serve(
gatherer_config) as gatherer_endpoint:
client_chain = make_pausing_beam_chain(
((0, PetersburgVM), ),
chain_id=999,
db=chaindb_fresh.db,
event_bus=pausing_endpoint,
loop=event_loop,
)
client = BeamSyncer(
client_chain,
chaindb_fresh.db,
AsyncChainDB(chaindb_fresh.db),
client_peer_pool,
gatherer_endpoint,
force_beam_block_number=beam_to_block,
checkpoint=checkpoint,
)
client_peer.logger.info("%s is serving churner blocks",
client_peer)
server_peer.logger.info("%s is syncing up churner blocks",
server_peer)
import_server = BlockImportServer(
pausing_endpoint,
client_chain,
token=client.cancel_token,
)
asyncio.ensure_future(import_server.run())
await pausing_endpoint.connect_to_endpoints(gatherer_config)
asyncio.ensure_future(client.run())
# We can sync at least 10 blocks in 1s at current speeds, (or reach the current one)
# Trying to keep the tests short-ish. A fuller test could always set the target header
# to the chaindb_churner canonical head, and increase the timeout significantly
target_block_number = min(beam_to_block + 10, 129)
target_head = chaindb_churner.get_canonical_block_header_by_number(
target_block_number)
await wait_for_head(chaindb_fresh, target_head, sync_timeout=10)
assert target_head.state_root in chaindb_fresh.db
# first stop the import server, so it doesn't hang waiting for state data
await import_server.cancel()
await client.cancel()
async def test_connect_to_endpoint(endpoint):
config = ConnectionConfig.from_name(generate_unique_name())
async with AsyncioEndpoint.serve(config):
await endpoint.connect_to_endpoints(config)
assert endpoint.is_connected_to(config.name)
def test_pickle_fails():
alice = AsyncioEndpoint("pickle-test")
with pytest.raises(Exception):
pickle.dumps(alice, protocol=pickle.HIGHEST_PROTOCOL)
async def test_beam_syncer(request, event_loop, event_bus, chaindb_fresh,
chaindb_churner, beam_to_block):
client_peer, server_peer = await get_directly_linked_peers(
request,
event_loop,
alice_headerdb=FakeAsyncHeaderDB(chaindb_fresh.db),
bob_headerdb=FakeAsyncHeaderDB(chaindb_churner.db))
# manually add endpoint for beam vm to make requests
pausing_config = ConnectionConfig.from_name("PausingEndpoint")
# manually add endpoint for trie data gatherer to serve requests
gatherer_config = ConnectionConfig.from_name("GathererEndpoint")
client_peer_pool = MockPeerPoolWithConnectedPeers([client_peer])
server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer],
event_bus=event_bus)
async with run_peer_pool_event_server(
event_bus, server_peer_pool,
handler_type=ETHPeerPoolEventServer), run_request_server(
event_bus,
FakeAsyncChainDB(chaindb_churner.db)), AsyncioEndpoint.serve(
pausing_config) as pausing_endpoint, AsyncioEndpoint.serve(
gatherer_config) as gatherer_endpoint:
BeamPetersburgVM = pausing_vm_decorator(PetersburgVM, pausing_endpoint)
class BeamPetersburgTestChain(FakeAsyncChain):
vm_configuration = ((0, BeamPetersburgVM), )
network_id = 999
client_chain = BeamPetersburgTestChain(chaindb_fresh.db)
client = BeamSyncer(
client_chain,
chaindb_fresh.db,
client_chain.chaindb,
client_peer_pool,
gatherer_endpoint,
beam_to_block,
)
client_peer.logger.info("%s is serving churner blocks", client_peer)
server_peer.logger.info("%s is syncing up churner blocks", server_peer)
import_server = BlockImportServer(pausing_endpoint,
client_chain,
token=client.cancel_token)
asyncio.ensure_future(import_server.run())
await pausing_endpoint.connect_to_endpoints(gatherer_config)
asyncio.ensure_future(client.run())
# We can sync at least 10 blocks in 1s at current speeds, (or reach the current one)
# Trying to keep the tests short-ish. A fuller test could always set the target header
# to the chaindb_churner canonical head, and increase the timeout significantly
target_block_number = min(beam_to_block + 10, 129)
target_head = chaindb_churner.get_canonical_block_header_by_number(
target_block_number)
await wait_for_head(chaindb_fresh, target_head, sync_timeout=4)
assert target_head.state_root in chaindb_fresh.db
# first stop the import server, so it doesn't hang waiting for state data
await import_server.cancel()
await client.cancel()
def test_pickle_fails():
alice = AsyncioEndpoint("pickle-test")
with pytest.raises(Exception):
pickle.dumps(alice)
async def _beam_syncing(
request,
event_loop,
event_bus,
chaindb_fresh,
chaindb_churner,
beam_to_block,
checkpoint=None,
VM_at_0=PetersburgVM,
enable_state_backfill=False,
):
client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db)
server_context = ChainContextFactory(headerdb__db=chaindb_churner.db)
peer_pair = LatestETHPeerPairFactory(
alice_peer_context=client_context,
bob_peer_context=server_context,
event_bus=event_bus,
)
backfiller = LatestETHPeerPairFactory(
alice_peer_context=client_context,
bob_peer_context=server_context,
event_bus=event_bus,
)
async with peer_pair as (client_peer,
server_peer), backfiller as (client2_peer,
backfill_peer):
# Need a name that will be unique per xdist-process, otherwise
# lahja IPC endpoints in each process will clobber each other
unique_process_name = uuid.uuid4()
# manually add endpoint for beam vm to make requests
pausing_config = ConnectionConfig.from_name(
f"PausingEndpoint-{unique_process_name}")
# manually add endpoint for trie data gatherer to serve requests
gatherer_config = ConnectionConfig.from_name(
f"GathererEndpoint-{unique_process_name}")
client_peer_pool = MockPeerPoolWithConnectedPeers(
[client_peer, backfill_peer],
event_bus=event_bus,
)
server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer],
event_bus=event_bus)
backfill_peer_pool = MockPeerPoolWithConnectedPeers(
[client2_peer], event_bus=event_bus)
async with run_peer_pool_event_server(
event_bus, server_peer_pool,
handler_type=ETHPeerPoolEventServer
), run_peer_pool_event_server(
event_bus, backfill_peer_pool,
handler_type=ETHPeerPoolEventServer
), background_asyncio_service(
ETHRequestServer(event_bus, TO_NETWORKING_BROADCAST_CONFIG,
AsyncChainDB(chaindb_churner.db))
), AsyncioEndpoint.serve(
pausing_config) as pausing_endpoint, AsyncioEndpoint.serve(
gatherer_config) as gatherer_endpoint:
client_chain = make_pausing_beam_chain(
((0, VM_at_0), ),
chain_id=999,
consensus_context_class=ConsensusContext,
db=chaindb_fresh.db,
event_bus=pausing_endpoint,
metrics_registry=NoopMetricsRegistry(),
loop=event_loop,
)
client = BeamSyncer(
client_chain,
chaindb_fresh.db,
AsyncChainDB(chaindb_fresh.db),
client_peer_pool,
gatherer_endpoint,
NoopMetricsRegistry(),
force_beam_block_number=beam_to_block,
checkpoint=checkpoint,
enable_state_backfill=enable_state_backfill,
enable_backfill=False,
)
client_peer.logger.info("%s is serving churner blocks",
client_peer)
backfill_peer.logger.info("%s is serving backfill state",
backfill_peer)
server_peer.logger.info("%s is syncing up churner blocks",
server_peer)
import_server = BlockImportServer(
pausing_endpoint,
client_chain,
)
async with background_asyncio_service(import_server):
await pausing_endpoint.connect_to_endpoints(gatherer_config)
async with background_asyncio_service(client):
yield client
async def test_connect_to_endpoint():
config = ConnectionConfig.from_name(generate_unique_name())
async with AsyncioEndpoint.serve(config):
async with AsyncioEndpoint("client").run() as client:
await client.connect_to_endpoint(config)
assert client.is_connected_to(config.name)