async def _sync(self, peer: ETHPeer) -> None:
head = await self.chaindb.coro_get_canonical_head()
head_td = await self.chaindb.coro_get_score(head.hash)
if peer.head_td <= head_td:
self.logger.info(
"Head TD (%d) announced by %s not higher than ours (%d), not syncing",
peer.head_td, peer, head_td)
return
self.logger.info("Starting sync with %s", peer)
# FIXME: Fetch a batch of headers, in reverse order, starting from our current head, and
# find the common ancestor between our chain and the peer's.
start_at = max(0, head.block_number - eth.MAX_HEADERS_FETCH)
while not self._sync_complete.is_set():
self.logger.info("Fetching chain segment starting at #%d", start_at)
peer.sub_proto.send_get_block_headers(start_at, eth.MAX_HEADERS_FETCH, reverse=False)
try:
headers = await wait_with_token(
self._new_headers.get(), peer.wait_until_finished(),
token=self.cancel_token,
timeout=self._reply_timeout)
except TimeoutError:
self.logger.warn("Timeout waiting for header batch from %s, aborting sync", peer)
await peer.stop()
break
if peer.is_finished():
self.logger.info("%s disconnected, aborting sync", peer)
break
self.logger.info("Got headers segment starting at #%d", start_at)
# TODO: Process headers for consistency.
head_number = await self._process_headers(peer, headers)
start_at = head_number + 1
async def _handle_msg(self, peer: ETHPeer, cmd: protocol.Command,
msg: protocol._DecodedMsgType) -> None:
if isinstance(cmd, eth.BlockHeaders):
msg = cast(List[BlockHeader], msg)
self.logger.debug("Got BlockHeaders from %d to %d",
msg[0].block_number, msg[-1].block_number)
self._new_headers.put_nowait(msg)
elif isinstance(cmd, eth.BlockBodies):
await self._handle_block_bodies(peer, cast(List[eth.BlockBody],
msg))
elif isinstance(cmd, eth.Receipts):
await self._handle_block_receipts(
peer, cast(List[List[eth.Receipt]], msg))
elif isinstance(cmd, eth.NewBlock):
msg = cast(Dict[str, Any], msg)
header = msg['block'][0]
actual_head = header.parent_hash
actual_td = msg['total_difficulty'] - header.difficulty
if actual_td > peer.head_td:
peer.head_hash = actual_head
peer.head_td = actual_td
self._sync_requests.put_nowait(peer)
elif isinstance(cmd, eth.Transactions):
# TODO: Figure out what to do with those.
pass
elif isinstance(cmd, eth.NewBlockHashes):
# TODO: Figure out what to do with those.
pass
else:
# TODO: There are other msg types we'll want to handle here, but for now just log them
# as a warning so we don't forget about it.
self.logger.warn("Got unexpected msg: %s (%s)", cmd, msg)
async def _handle_new_block(self, peer: ETHPeer, msg: Dict[str, Any]) -> None:
header = msg['block'][0]
actual_head = header.parent_hash
actual_td = msg['total_difficulty'] - header.difficulty
if actual_td > peer.head_td:
peer.head_hash = actual_head
peer.head_td = actual_td
self._sync_requests.put_nowait(peer)
async def _sync(self, peer: ETHPeer) -> None:
head = await self.chaindb.coro_get_canonical_head()
head_td = await self.chaindb.coro_get_score(head.hash)
if peer.head_td <= head_td:
self.logger.debug(
"Head TD (%d) announced by %s not higher than ours (%d), not syncing",
peer.head_td, peer, head_td)
return
# FIXME: Fetch a batch of headers, in reverse order, starting from our current head, and
# find the common ancestor between our chain and the peer's.
start_at = max(0, head.block_number - eth.MAX_HEADERS_FETCH)
self.logger.debug("Asking %s for header batch starting at %d", peer, start_at)
peer.sub_proto.send_get_block_headers(start_at, eth.MAX_HEADERS_FETCH, reverse=False)
max_consecutive_timeouts = 3
consecutive_timeouts = 0
while True:
try:
headers = await wait_with_token(
self._new_headers.get(), peer.wait_until_finished(),
token=self.cancel_token,
timeout=3)
except OperationCancelled:
break
except TimeoutError:
self.logger.debug("Timeout waiting for header batch from %s", peer)
consecutive_timeouts += 1
if consecutive_timeouts > max_consecutive_timeouts:
self.logger.debug(
"Too many consecutive timeouts waiting for header batch, aborting sync "
"with %s", peer)
break
continue
if peer.is_finished():
self.logger.debug("%s disconnected, stopping sync", peer)
break
consecutive_timeouts = 0
if headers[-1].block_number <= start_at:
self.logger.debug(
"Ignoring headers from %d to %d as they've been processed already",
headers[0].block_number, headers[-1].block_number)
continue
# TODO: Process headers for consistency.
for header in headers:
await self.chaindb.coro_persist_header(header)
start_at = header.block_number
self._body_requests.put_nowait(headers)
self._receipt_requests.put_nowait(headers)
self.logger.debug("Asking %s for header batch starting at %d", peer, start_at)
# TODO: Instead of requesting sequential batches from a single peer, request a header
# skeleton and make concurrent requests, using as many peers as possible, to fill the
# skeleton.
peer.sub_proto.send_get_block_headers(start_at, eth.MAX_HEADERS_FETCH, reverse=False)
async def handle_msg(self, peer: ETHPeer, cmd: protocol.Command,
msg: protocol._DecodedMsgType) -> None:
loop = asyncio.get_event_loop()
if isinstance(cmd, eth.BlockHeaders):
msg = cast(List[BlockHeader], msg)
self.logger.debug(
"Got BlockHeaders from %d to %d", msg[0].block_number, msg[-1].block_number)
self._new_headers.put_nowait(msg)
elif isinstance(cmd, eth.BlockBodies):
msg = cast(List[eth.BlockBody], msg)
self.logger.debug("Got %d BlockBodies from %s", len(msg), peer)
iterator = map(make_trie_root_and_nodes, [body.transactions for body in msg])
transactions_tries = await loop.run_in_executor(
self.executor, list, iterator) # type: List[Tuple[bytes, Any]]
for i in range(len(msg)):
body = msg[i]
tx_root, trie_dict_data = transactions_tries[i]
await self.chaindb.coro_persist_trie_data_dict(trie_dict_data)
# TODO: Add transactions to canonical chain; blocked by
# https://github.com/ethereum/py-evm/issues/337
uncles_hash = await self.chaindb.coro_persist_uncles(body.uncles)
self._pending_bodies.pop((tx_root, uncles_hash), None)
elif isinstance(cmd, eth.Receipts):
msg = cast(List[List[eth.Receipt]], msg)
self.logger.debug("Got Receipts for %d blocks from %s", len(msg), peer)
iterator = map(make_trie_root_and_nodes, msg)
receipts_tries = await loop.run_in_executor(
self.executor, list, iterator) # type: List[Tuple[bytes, Any]]
for receipt_root, trie_dict_data in receipts_tries:
if receipt_root not in self._pending_receipts:
self.logger.warning(
"Got unexpected receipt root: %s",
encode_hex(receipt_root),
)
continue
await self.chaindb.coro_persist_trie_data_dict(trie_dict_data)
self._pending_receipts.pop(receipt_root)
elif isinstance(cmd, eth.NewBlock):
msg = cast(Dict[str, Any], msg)
header = msg['block'][0]
actual_head = header.parent_hash
actual_td = msg['total_difficulty'] - header.difficulty
if actual_td > peer.head_td:
peer.head_hash = actual_head
peer.head_td = actual_td
self._sync_requests.put_nowait(peer)
elif isinstance(cmd, eth.Transactions):
# TODO: Figure out what to do with those.
pass
elif isinstance(cmd, eth.NewBlockHashes):
# TODO: Figure out what to do with those.
pass
else:
# TODO: There are other msg types we'll want to handle here, but for now just log them
# as a warning so we don't forget about it.
self.logger.warn("Got unexpected msg: %s (%s)", cmd, msg)
async def _sync(self, peer: ETHPeer) -> None:
head = await self.chaindb.coro_get_canonical_head()
head_td = await self.chaindb.coro_get_score(head.hash)
if peer.head_td <= head_td:
self.logger.info(
"Head TD (%d) announced by %s not higher than ours (%d), not syncing",
peer.head_td, peer, head_td)
return
# FIXME: Fetch a batch of headers, in reverse order, starting from our current head, and
# find the common ancestor between our chain and the peer's.
start_at = max(0, head.block_number - eth.MAX_HEADERS_FETCH)
self.logger.debug("Asking %s for header batch starting at %d", peer,
start_at)
peer.sub_proto.send_get_block_headers(start_at,
eth.MAX_HEADERS_FETCH,
reverse=False)
while True:
# TODO: Consider stalling header fetching if there are more than X blocks/receipts
# pending, to avoid timeouts caused by us not being able to process (decode/store)
# blocks/receipts fast enough.
try:
headers = await wait_with_token(self._new_headers.get(),
peer.wait_until_finished(),
token=self.cancel_token,
timeout=self._reply_timeout)
except OperationCancelled:
break
except TimeoutError:
self.logger.warn(
"Timeout waiting for header batch from %s, aborting sync",
peer)
await peer.stop()
break
if peer.is_finished():
self.logger.info("%s disconnected, aborting sync", peer)
break
# TODO: Process headers for consistency.
for header in headers:
await self.chaindb.coro_persist_header(header)
start_at = header.block_number + 1
self._body_requests.put_nowait(headers)
self._receipt_requests.put_nowait(headers)
self.logger.debug("Asking %s for header batch starting at %d",
peer, start_at)
# TODO: Instead of requesting sequential batches from a single peer, request a header
# skeleton and make concurrent requests, using as many peers as possible, to fill the
# skeleton.
peer.sub_proto.send_get_block_headers(start_at,
eth.MAX_HEADERS_FETCH,
reverse=False)
async def handle_msg(self, peer: ETHPeer, cmd: protocol.Command,
msg: protocol._DecodedMsgType) -> None:
if isinstance(cmd, eth.BlockHeaders):
msg = cast(List[BlockHeader], msg)
self.logger.debug("Got BlockHeaders from %d to %d",
msg[0].block_number, msg[-1].block_number)
self._new_headers.put_nowait(msg)
elif isinstance(cmd, eth.BlockBodies):
msg = cast(List[eth.BlockBody], msg)
self.logger.debug("Got %d BlockBodies", len(msg))
for body in msg:
tx_root, trie_dict_data = make_trie_root_and_nodes(
body.transactions)
await self.chaindb.coro_persist_trie_data_dict(trie_dict_data)
# TODO: Add transactions to canonical chain; blocked by
# https://github.com/ethereum/py-evm/issues/337
uncles_hash = await self.chaindb.coro_persist_uncles(
body.uncles)
self._pending_bodies.pop((tx_root, uncles_hash), None)
elif isinstance(cmd, eth.Receipts):
msg = cast(List[List[eth.Receipt]], msg)
self.logger.debug("Got Receipts for %d blocks", len(msg))
for block_receipts in msg:
receipt_root, trie_dict_data = make_trie_root_and_nodes(
block_receipts)
await self.chaindb.coro_persist_trie_data_dict(trie_dict_data)
self._pending_receipts.pop(receipt_root, None)
elif isinstance(cmd, eth.NewBlock):
msg = cast(Dict[str, Any], msg)
header = msg['block'][0]
actual_head = header.parent_hash
actual_td = msg['total_difficulty'] - header.difficulty
if actual_td > peer.head_td:
peer.head_hash = actual_head
peer.head_td = actual_td
self._sync_requests.put_nowait(peer)
elif isinstance(cmd, eth.Transactions):
# TODO: Figure out what to do with those.
pass
elif isinstance(cmd, eth.NewBlockHashes):
# TODO: Figure out what to do with those.
pass
else:
# TODO: There are other msg types we'll want to handle here, but for now just log them
# as a warning so we don't forget about it.
self.logger.warn("Got unexpected msg: %s (%s)", cmd, msg)
async def receive_handshake(self, reader: asyncio.StreamReader,
writer: asyncio.StreamWriter) -> None:
# Use reader to read the auth_init msg until EOF
msg = await reader.read(ENCRYPTED_AUTH_MSG_LEN)
# Use HandshakeResponder.decode_authentication(auth_init_message) on auth init msg
try:
ephem_pubkey, initiator_nonce, initiator_pubkey = decode_authentication(
msg, self.privkey)
# Try to decode as EIP8
except DecryptionError:
msg_size = big_endian_to_int(msg[:2])
remaining_bytes = msg_size - ENCRYPTED_AUTH_MSG_LEN + 2
msg += await reader.read(remaining_bytes)
ephem_pubkey, initiator_nonce, initiator_pubkey = decode_authentication(
msg, self.privkey)
# Get remote's address: IPv4 or IPv6
ip, port, *_ = writer.get_extra_info("peername")
remote_address = Address(ip, port)
# Create `HandshakeResponder(remote: kademlia.Node, privkey: datatypes.PrivateKey)` instance
initiator_remote = Node(initiator_pubkey, remote_address)
responder = HandshakeResponder(initiator_remote, self.privkey)
# Call `HandshakeResponder.create_auth_ack_message(nonce: bytes)` to create the reply
responder_nonce = secrets.token_bytes(HASH_LEN)
auth_ack_msg = responder.create_auth_ack_message(nonce=responder_nonce)
auth_ack_ciphertext = responder.encrypt_auth_ack_message(auth_ack_msg)
# Use the `writer` to send the reply to the remote
writer.write(auth_ack_ciphertext)
await writer.drain()
# Call `HandshakeResponder.derive_shared_secrets()` and use return values to create `Peer`
aes_secret, mac_secret, egress_mac, ingress_mac = responder.derive_secrets(
initiator_nonce=initiator_nonce,
responder_nonce=responder_nonce,
remote_ephemeral_pubkey=ephem_pubkey,
auth_init_ciphertext=msg,
auth_ack_ciphertext=auth_ack_ciphertext)
# Create and register peer in peer_pool
eth_peer = ETHPeer(remote=initiator_remote,
privkey=self.privkey,
reader=reader,
writer=writer,
aes_secret=aes_secret,
mac_secret=mac_secret,
egress_mac=egress_mac,
ingress_mac=ingress_mac,
chaindb=self.chaindb,
network_id=self.network_id)
self.peer_pool.add_peer(eth_peer)
async def _sync(self, peer: ETHPeer) -> None:
head = await self.chaindb.coro_get_canonical_head()
head_td = await self.chaindb.coro_get_score(head.hash)
if peer.head_td <= head_td:
self.logger.info(
"Head TD (%d) announced by %s not higher than ours (%d), not syncing",
peer.head_td, peer, head_td)
return
self.logger.info("Starting sync with %s", peer)
# FIXME: Fetch a batch of headers, in reverse order, starting from our current head, and
# find the common ancestor between our chain and the peer's.
start_at = max(0, head.block_number - eth.MAX_HEADERS_FETCH)
while True:
self.logger.info("Fetching chain segment starting at #%d",
start_at)
peer.sub_proto.send_get_block_headers(start_at,
eth.MAX_HEADERS_FETCH,
reverse=False)
try:
headers = await wait_with_token(self._new_headers.get(),
peer.wait_until_finished(),
token=self.cancel_token,
timeout=self._reply_timeout)
except TimeoutError:
self.logger.warn(
"Timeout waiting for header batch from %s, aborting sync",
peer)
await peer.stop()
break
if peer.is_finished():
self.logger.info("%s disconnected, aborting sync", peer)
break
self.logger.info("Got headers segment starting at #%d", start_at)
# TODO: Process headers for consistency.
await self._download_block_parts(
[header for header in headers if not _is_body_empty(header)],
self.request_bodies, self._downloaded_bodies, _body_key,
'body')
self.logger.info(
"Got block bodies for chain segment starting at #%d", start_at)
missing_receipts = [
header for header in headers if not _is_receipts_empty(header)
]
# Post-Byzantium blocks may have identical receipt roots (e.g. when they have the same
# number of transactions and all succeed/failed: ropsten blocks 2503212 and 2503284),
# so we do this to avoid requesting the same receipts multiple times.
missing_receipts = list(unique(missing_receipts,
key=_receipts_key))
await self._download_block_parts(missing_receipts,
self.request_receipts,
self._downloaded_receipts,
_receipts_key, 'receipt')
self.logger.info(
"Got block receipts for chain segment starting at #%d",
start_at)
for header in headers:
await self.chaindb.coro_persist_header(header)
start_at = header.block_number + 1
self.logger.info("Imported chain segment, new head: #%d",
start_at - 1)
head = await self.chaindb.coro_get_canonical_head()
if head.hash == peer.head_hash:
self.logger.info("Chain sync with %s completed", peer)
self._sync_complete.set()
break
