async def test_token_wait(event_loop):
token = CancelToken('token')
event_loop.call_soon(token.trigger)
done, pending = await asyncio.wait([token.wait()], timeout=0.1)
assert len(done) == 1
assert len(pending) == 0
assert token.triggered
async def test_token_wait(event_loop):
token = CancelToken('token')
event_loop.call_soon(token.trigger)
done, pending = await asyncio.wait([token.wait()], timeout=0.1)
assert len(done) == 1
assert len(pending) == 0
assert token.triggered
async def test_wait_cancel_pending_tasks_on_cancellation(event_loop):
"""Test that cancelling a pending CancelToken.wait() coroutine doesn't leave .wait()
coroutines for any chained tokens behind.
"""
token = CancelToken('token').chain(CancelToken('token2')).chain(CancelToken('token3'))
token_wait_coroutine = token.wait()
done, pending = await asyncio.wait([token_wait_coroutine], timeout=0.1)
assert len(done) == 0
assert len(pending) == 1
pending_task = pending.pop()
assert pending_task._coro == token_wait_coroutine
pending_task.cancel()
await assert_only_current_task_not_done()
async def test_wait_cancel_pending_tasks_on_cancellation(event_loop):
"""Test that cancelling a pending CancelToken.wait() coroutine doesn't leave .wait()
coroutines for any chained tokens behind.
"""
token = CancelToken('token').chain(CancelToken('token2')).chain(CancelToken('token3'))
token_wait_coroutine = token.wait()
done, pending = await asyncio.wait([token_wait_coroutine], timeout=0.1)
assert len(done) == 0
assert len(pending) == 1
pending_task = pending.pop()
assert pending_task._coro == token_wait_coroutine
pending_task.cancel()
await assert_only_current_task_not_done()
class PeerPool:
"""PeerPool attempts to keep connections to at least min_peers on the given network."""
logger = logging.getLogger("p2p.peer.PeerPool")
_connect_loop_sleep = 2
_last_lookup = 0 # type: float
_lookup_interval = 5 # type: int
def __init__(self,
peer_class: Type[BasePeer],
chaindb: AsyncChainDB,
network_id: int,
privkey: datatypes.PrivateKey,
discovery: DiscoveryProtocol,
min_peers: int = DEFAULT_MIN_PEERS,
) -> None:
self.peer_class = peer_class
self.chaindb = chaindb
self.network_id = network_id
self.privkey = privkey
self.discovery = discovery
self.min_peers = min_peers
self.connected_nodes = {} # type: Dict[Node, BasePeer]
self.cancel_token = CancelToken('PeerPool')
self._subscribers = [] # type: List[PeerPoolSubscriber]
def get_nodes_to_connect(self) -> Generator[Node, None, None]:
return self.discovery.get_random_nodes(self.min_peers)
def subscribe(self, subscriber: PeerPoolSubscriber) -> None:
self._subscribers.append(subscriber)
for peer in self.connected_nodes.values():
subscriber.register_peer(peer)
def unsubscribe(self, subscriber: PeerPoolSubscriber) -> None:
if subscriber in self._subscribers:
self._subscribers.remove(subscriber)
def start_peer(self, peer):
asyncio.ensure_future(peer.run(finished_callback=self._peer_finished))
self.add_peer(peer)
def add_peer(self, peer):
self.logger.debug('Adding peer (%s) ...', str(peer))
self.connected_nodes[peer.remote] = peer
self.logger.debug('Number of peers: %d', len(self.connected_nodes))
for subscriber in self._subscribers:
subscriber.register_peer(peer)
async def run(self) -> None:
self.logger.info("Running PeerPool...")
while not self.cancel_token.triggered:
try:
await self.maybe_connect_to_more_peers()
except OperationCancelled as e:
self.logger.debug("PeerPool finished: %s", e)
break
except: # noqa: E722
# Most unexpected errors should be transient, so we log and restart from scratch.
self.logger.exception("Unexpected error, restarting")
await self.stop_all_peers()
# Wait self._connect_loop_sleep seconds, unless we're asked to stop.
await asyncio.wait([self.cancel_token.wait()], timeout=self._connect_loop_sleep)
async def stop_all_peers(self) -> None:
self.logger.info("Stopping all peers ...")
await asyncio.gather(
*[peer.stop() for peer in self.connected_nodes.values()])
async def stop(self) -> None:
self.cancel_token.trigger()
await self.stop_all_peers()
async def connect(self, remote: Node) -> BasePeer:
"""
Connect to the given remote and return a Peer instance when successful.
Returns None if the remote is unreachable, times out or is useless.
"""
if remote in self.connected_nodes:
self.logger.debug("Skipping %s; already connected to it", remote)
return None
expected_exceptions = (
UnreachablePeer, TimeoutError, PeerConnectionLost, HandshakeFailure)
try:
self.logger.debug("Connecting to %s...", remote)
peer = await wait_with_token(
handshake(remote, self.privkey, self.peer_class, self.chaindb, self.network_id),
token=self.cancel_token,
timeout=HANDSHAKE_TIMEOUT)
return peer
except OperationCancelled:
# Pass it on to instruct our main loop to stop.
raise
except expected_exceptions as e:
self.logger.debug("Could not complete handshake with %s: %s", remote, repr(e))
except Exception:
self.logger.exception("Unexpected error during auth/p2p handshake with %s", remote)
return None
async def lookup_random_node(self) -> None:
# This method runs in the background, so we must catch OperationCancelled here otherwise
# asyncio will warn that its exception was never retrieved.
try:
await self.discovery.lookup_random(self.cancel_token)
except OperationCancelled:
pass
self._last_lookup = time.time()
async def maybe_connect_to_more_peers(self) -> None:
"""Connect to more peers if we're not yet connected to at least self.min_peers."""
if len(self.connected_nodes) >= self.min_peers:
self.logger.debug(
"Already connected to %s peers: %s; sleeping",
len(self.connected_nodes),
[remote for remote in self.connected_nodes])
return
if self._last_lookup + self._lookup_interval < time.time():
asyncio.ensure_future(self.lookup_random_node())
await self._connect_to_nodes(self.get_nodes_to_connect())
# In some cases (e.g ROPSTEN or private testnets), the discovery table might be full of
# bad peers so if we can't connect to any peers we try a random bootstrap node as well.
if not self.peers:
await self._connect_to_nodes(self._get_random_bootnode())
def _get_random_bootnode(self) -> Generator[Node, None, None]:
if self.discovery.bootstrap_nodes:
yield random.choice(self.discovery.bootstrap_nodes)
else:
self.logger.warning('No bootstrap_nodes')
async def _connect_to_nodes(self, nodes: Generator[Node, None, None]) -> None:
for node in nodes:
# TODO: Consider changing connect() to raise an exception instead of returning None,
# as discussed in
# https://github.com/ethereum/py-evm/pull/139#discussion_r152067425
peer = await self.connect(node)
if peer is not None:
self.logger.info("Successfully connected to %s", peer)
self.start_peer(peer)
if len(self.connected_nodes) >= self.min_peers:
return
def _peer_finished(self, peer: BasePeer) -> None:
"""Remove the given peer from our list of connected nodes.
This is passed as a callback to be called when a peer finishes.
"""
if peer.remote in self.connected_nodes:
self.connected_nodes.pop(peer.remote)
@property
def peers(self) -> List[BasePeer]:
peers = list(self.connected_nodes.values())
# Shuffle the list of peers so that dumb callsites are less likely to send all requests to
# a single peer even if they always pick the first one from the list.
random.shuffle(peers)
return peers
async def get_random_peer(self) -> BasePeer:
while not self.peers:
self.logger.debug("No connected peers, sleeping a bit")
await asyncio.sleep(0.5)
return random.choice(self.peers)
class PeerPool:
"""PeerPool attempts to keep connections to at least min_peers on the given network."""
logger = logging.getLogger("p2p.peer.PeerPool")
min_peers = 2
_connect_loop_sleep = 2
def __init__(
self,
peer_class: Type[BasePeer],
chaindb: ChainDB,
network_id: int,
privkey: datatypes.PrivateKey,
) -> None:
self.peer_class = peer_class
self.chaindb = chaindb
self.network_id = network_id
self.privkey = privkey
self.connected_nodes = {} # type: Dict[Node, BasePeer]
self.cancel_token = CancelToken('PeerPool')
self._subscribers = [] # type: List[PeerPoolSubscriber]
def subscribe(self, subscriber: PeerPoolSubscriber) -> None:
self._subscribers.append(subscriber)
for peer in self.connected_nodes.values():
subscriber.register_peer(peer)
def unsubscribe(self, subscriber: PeerPoolSubscriber) -> None:
if subscriber in self._subscribers:
self._subscribers.remove(subscriber)
async def get_nodes_to_connect(self) -> List[Node]:
# TODO: This should use the Discovery service to lookup nodes to connect to, but our
# current implementation only supports v4 and with that it takes an insane amount of time
# to find any LES nodes with the same network ID as us, so for now we hard-code some nodes
# that seem to have a good uptime.
from evm.chains.ropsten import RopstenChain
from evm.chains.mainnet import MainnetChain
if self.network_id == MainnetChain.network_id:
return [
Node(
keys.PublicKey(
decode_hex(
"1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082"
)), # noqa: E501
Address("52.74.57.123", 30303, 30303)),
Node(
keys.PublicKey(
decode_hex(
"78de8a0916848093c73790ead81d1928bec737d565119932b98c6b100d944b7a95e94f847f689fc723399d2e31129d182f7ef3863f2b4c820abbf3ab2722344d"
)), # noqa: E501
Address("191.235.84.50", 30303, 30303)),
Node(
keys.PublicKey(
decode_hex(
"ddd81193df80128880232fc1deb45f72746019839589eeb642d3d44efbb8b2dda2c1a46a348349964a6066f8afb016eb2a8c0f3c66f32fadf4370a236a4b5286"
)), # noqa: E501
Address("52.231.202.145", 30303, 30303)),
Node(
keys.PublicKey(
decode_hex(
"3f1d12044546b76342d59d4a05532c14b85aa669704bfe1f864fe079415aa2c02d743e03218e57a33fb94523adb54032871a6c51b2cc5514cb7c7e35b3ed0a99"
)), # noqa: E501
Address("13.93.211.84", 30303, 30303)),
]
elif self.network_id == RopstenChain.network_id:
return [
Node(
keys.PublicKey(
decode_hex(
"88c2b24429a6f7683fbfd06874ae3f1e7c8b4a5ffb846e77c705ba02e2543789d66fc032b6606a8d8888eb6239a2abe5897ce83f78dcdcfcb027d6ea69aa6fe9"
)), # noqa: E501
Address("163.172.157.61", 30303, 30303)),
Node(
keys.PublicKey(
decode_hex(
"a1ef9ba5550d5fac27f7cbd4e8d20a643ad75596f307c91cd6e7f85b548b8a6bf215cca436d6ee436d6135f9fe51398f8dd4c0bd6c6a0c332ccb41880f33ec12"
)), # noqa: E501
Address("51.15.218.125", 30303, 30303)),
Node(
keys.PublicKey(
decode_hex(
"e80276aabb7682a4a659f4341c1199de79d91a2e500a6ee9bed16ed4ce927ba8d32ba5dea357739ffdf2c5bcc848d3064bb6f149f0b4249c1f7e53f8bf02bfc8"
)), # noqa: E501
Address("51.15.39.57", 30303, 30303)),
Node(
keys.PublicKey(
decode_hex(
"584c0db89b00719e9e7b1b5c32a4a8942f379f4d5d66bb69f9c7fa97fa42f64974e7b057b35eb5a63fd7973af063f9a1d32d8c60dbb4854c64cb8ab385470258"
)), # noqa: E501
Address("51.15.35.2", 30303, 30303)),
Node(
keys.PublicKey(
decode_hex(
"d40871fc3e11b2649700978e06acd68a24af54e603d4333faecb70926ca7df93baa0b7bf4e927fcad9a7c1c07f9b325b22f6d1730e728314d0e4e6523e5cebc2"
)), # noqa: E501
Address("51.15.132.235", 30303, 30303)),
Node(
keys.PublicKey(
decode_hex(
"482484b9198530ee2e00db89791823244ca41dcd372242e2e1297dd06f6d8dd357603960c5ad9cc8dc15fcdf0e4edd06b7ad7db590e67a0b54f798c26581ebd7"
)), # noqa: E501
Address("51.15.75.138", 30303, 30303)),
]
else:
raise ValueError("Unknown network_id: {}".format(self.network_id))
async def run(self):
self.logger.info("Running PeerPool...")
while not self.cancel_token.triggered:
try:
await self.maybe_connect_to_more_peers()
except: # noqa: E722
# Most unexpected errors should be transient, so we log and restart from scratch.
self.logger.error("Unexpected error (%s), restarting",
traceback.format_exc())
await self.stop_all_peers()
# Wait self._connect_loop_sleep seconds, unless we're asked to stop.
await asyncio.wait([self.cancel_token.wait()],
timeout=self._connect_loop_sleep)
async def stop_all_peers(self):
self.logger.info("Stopping all peers ...")
await asyncio.gather(
*[peer.stop() for peer in self.connected_nodes.values()])
async def stop(self):
self.cancel_token.trigger()
await self.stop_all_peers()
async def connect(self, remote: Node) -> BasePeer:
"""
Connect to the given remote and return a Peer instance when successful.
Returns None if the remote is unreachable, times out or is useless.
"""
if remote in self.connected_nodes:
self.logger.debug("Skipping %s; already connected to it", remote)
return None
expected_exceptions = (UnreachablePeer, asyncio.TimeoutError,
PeerConnectionLost, HandshakeFailure)
try:
self.logger.info("Connecting to %s...", remote)
# TODO: Use asyncio.wait() and our cancel_token here to cancel in case the token is
# triggered.
peer = await asyncio.wait_for(
handshake(remote, self.privkey, self.peer_class, self.chaindb,
self.network_id), HANDSHAKE_TIMEOUT)
return peer
except expected_exceptions as e:
self.logger.info("Could not complete handshake with %s: %s",
remote, repr(e))
except Exception:
self.logger.warning(
"Unexpected error during auth/p2p handshake with %s: %s",
remote, traceback.format_exc())
return None
async def maybe_connect_to_more_peers(self):
"""Connect to more peers if we're not yet connected to at least self.min_peers."""
if len(self.connected_nodes) >= self.min_peers:
self.logger.debug("Already connected to %s peers: %s; sleeping",
len(self.connected_nodes),
[remote for remote in self.connected_nodes])
return
for node in await self.get_nodes_to_connect():
# TODO: Consider changing connect() to raise an exception instead of returning None,
# as discussed in
# https://github.com/pipermerriam/py-evm/pull/139#discussion_r152067425
peer = await self.connect(node)
if peer is not None:
self.logger.info("Successfully connected to %s", peer)
asyncio.ensure_future(
peer.run(finished_callback=self._peer_finished))
self.connected_nodes[peer.remote] = peer
for subscriber in self._subscribers:
subscriber.register_peer(peer)
def _peer_finished(self, peer: BasePeer) -> None:
"""Remove the given peer from our list of connected nodes.
This is passed as a callback to be called when a peer finishes.
"""
if peer.remote in self.connected_nodes:
self.connected_nodes.pop(peer.remote)
@property
def peers(self) -> List[BasePeer]:
return list(self.connected_nodes.values())
class PeerPool:
"""PeerPool attempts to keep connections to at least min_peers on the given network."""
logger = logging.getLogger("p2p.peer.PeerPool")
min_peers = 2
_connect_loop_sleep = 2
def __init__(self,
peer_class: Type[BasePeer],
chaindb: AsyncChainDB,
network_id: int,
privkey: datatypes.PrivateKey,
) -> None:
self.peer_class = peer_class
self.chaindb = chaindb
self.network_id = network_id
self.privkey = privkey
self.connected_nodes = {} # type: Dict[Node, BasePeer]
self.cancel_token = CancelToken('PeerPool')
self._subscribers = [] # type: List[PeerPoolSubscriber]
def subscribe(self, subscriber: PeerPoolSubscriber) -> None:
self._subscribers.append(subscriber)
for peer in self.connected_nodes.values():
subscriber.register_peer(peer)
def unsubscribe(self, subscriber: PeerPoolSubscriber) -> None:
if subscriber in self._subscribers:
self._subscribers.remove(subscriber)
async def get_nodes_to_connect(self) -> List[Node]:
# TODO: This should use the Discovery service to lookup nodes to connect to, but our
# current implementation only supports v4 and with that it takes an insane amount of time
# to find any LES nodes with the same network ID as us, so for now we hard-code some nodes
# that seem to have a good uptime.
from evm.chains.ropsten import RopstenChain
from evm.chains.mainnet import MainnetChain
if self.network_id == MainnetChain.network_id:
return [
Node(
keys.PublicKey(decode_hex("1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082")), # noqa: E501
Address("52.74.57.123", 30303, 30303)),
Node(
keys.PublicKey(decode_hex("78de8a0916848093c73790ead81d1928bec737d565119932b98c6b100d944b7a95e94f847f689fc723399d2e31129d182f7ef3863f2b4c820abbf3ab2722344d")), # noqa: E501
Address("191.235.84.50", 30303, 30303)),
Node(
keys.PublicKey(decode_hex("ddd81193df80128880232fc1deb45f72746019839589eeb642d3d44efbb8b2dda2c1a46a348349964a6066f8afb016eb2a8c0f3c66f32fadf4370a236a4b5286")), # noqa: E501
Address("52.231.202.145", 30303, 30303)),
Node(
keys.PublicKey(decode_hex("3f1d12044546b76342d59d4a05532c14b85aa669704bfe1f864fe079415aa2c02d743e03218e57a33fb94523adb54032871a6c51b2cc5514cb7c7e35b3ed0a99")), # noqa: E501
Address("13.93.211.84", 30303, 30303)),
]
elif self.network_id == RopstenChain.network_id:
return [
Node(
keys.PublicKey(decode_hex("88c2b24429a6f7683fbfd06874ae3f1e7c8b4a5ffb846e77c705ba02e2543789d66fc032b6606a8d8888eb6239a2abe5897ce83f78dcdcfcb027d6ea69aa6fe9")), # noqa: E501
Address("163.172.157.61", 30303, 30303)),
Node(
keys.PublicKey(decode_hex("a1ef9ba5550d5fac27f7cbd4e8d20a643ad75596f307c91cd6e7f85b548b8a6bf215cca436d6ee436d6135f9fe51398f8dd4c0bd6c6a0c332ccb41880f33ec12")), # noqa: E501
Address("51.15.218.125", 30303, 30303)),
Node(
keys.PublicKey(decode_hex("e80276aabb7682a4a659f4341c1199de79d91a2e500a6ee9bed16ed4ce927ba8d32ba5dea357739ffdf2c5bcc848d3064bb6f149f0b4249c1f7e53f8bf02bfc8")), # noqa: E501
Address("51.15.39.57", 30303, 30303)),
Node(
keys.PublicKey(decode_hex("584c0db89b00719e9e7b1b5c32a4a8942f379f4d5d66bb69f9c7fa97fa42f64974e7b057b35eb5a63fd7973af063f9a1d32d8c60dbb4854c64cb8ab385470258")), # noqa: E501
Address("51.15.35.2", 30303, 30303)),
Node(
keys.PublicKey(decode_hex("d40871fc3e11b2649700978e06acd68a24af54e603d4333faecb70926ca7df93baa0b7bf4e927fcad9a7c1c07f9b325b22f6d1730e728314d0e4e6523e5cebc2")), # noqa: E501
Address("51.15.132.235", 30303, 30303)),
Node(
keys.PublicKey(decode_hex("482484b9198530ee2e00db89791823244ca41dcd372242e2e1297dd06f6d8dd357603960c5ad9cc8dc15fcdf0e4edd06b7ad7db590e67a0b54f798c26581ebd7")), # noqa: E501
Address("51.15.75.138", 30303, 30303)),
]
else:
raise ValueError("Unknown network_id: {}".format(self.network_id))
async def run(self):
self.logger.info("Running PeerPool...")
while not self.cancel_token.triggered:
try:
await self.maybe_connect_to_more_peers()
except: # noqa: E722
# Most unexpected errors should be transient, so we log and restart from scratch.
self.logger.error("Unexpected error (%s), restarting", traceback.format_exc())
await self.stop_all_peers()
# Wait self._connect_loop_sleep seconds, unless we're asked to stop.
await asyncio.wait([self.cancel_token.wait()], timeout=self._connect_loop_sleep)
async def stop_all_peers(self):
self.logger.info("Stopping all peers ...")
await asyncio.gather(
*[peer.stop() for peer in self.connected_nodes.values()])
async def stop(self):
self.cancel_token.trigger()
await self.stop_all_peers()
async def connect(self, remote: Node) -> BasePeer:
"""
Connect to the given remote and return a Peer instance when successful.
Returns None if the remote is unreachable, times out or is useless.
"""
if remote in self.connected_nodes:
self.logger.debug("Skipping %s; already connected to it", remote)
return None
expected_exceptions = (
UnreachablePeer, asyncio.TimeoutError, PeerConnectionLost, HandshakeFailure)
try:
self.logger.info("Connecting to %s...", remote)
# TODO: Use asyncio.wait() and our cancel_token here to cancel in case the token is
# triggered.
peer = await asyncio.wait_for(
handshake(remote, self.privkey, self.peer_class, self.chaindb, self.network_id),
HANDSHAKE_TIMEOUT)
return peer
except expected_exceptions as e:
self.logger.info("Could not complete handshake with %s: %s", remote, repr(e))
except Exception:
self.logger.warning("Unexpected error during auth/p2p handshake with %s: %s",
remote, traceback.format_exc())
return None
async def maybe_connect_to_more_peers(self):
"""Connect to more peers if we're not yet connected to at least self.min_peers."""
if len(self.connected_nodes) >= self.min_peers:
self.logger.debug(
"Already connected to %s peers: %s; sleeping",
len(self.connected_nodes),
[remote for remote in self.connected_nodes])
return
for node in await self.get_nodes_to_connect():
# TODO: Consider changing connect() to raise an exception instead of returning None,
# as discussed in
# https://github.com/pipermerriam/py-evm/pull/139#discussion_r152067425
peer = await self.connect(node)
if peer is not None:
self.logger.info("Successfully connected to %s", peer)
asyncio.ensure_future(peer.run(finished_callback=self._peer_finished))
self.connected_nodes[peer.remote] = peer
for subscriber in self._subscribers:
subscriber.register_peer(peer)
def _peer_finished(self, peer: BasePeer) -> None:
"""Remove the given peer from our list of connected nodes.
This is passed as a callback to be called when a peer finishes.
"""
if peer.remote in self.connected_nodes:
self.connected_nodes.pop(peer.remote)
@property
def peers(self) -> List[BasePeer]:
return list(self.connected_nodes.values())
