class Disconnect(BaseCommand[DisconnectReason]):
protocol_command_id = 1
serialization_codec = RLPCodec(
sedes=DISCONNECT_STRUCTURE,
process_outbound_payload_fn=compose(lambda v: (v, ),
operator.attrgetter('value')),
process_inbound_payload_fn=compose(DisconnectReason,
operator.itemgetter(0)),
)
def get_request_formatters(
method_name: Union[RPCEndpoint, Callable[..., RPCEndpoint]]
) -> Dict[str, Callable[..., Any]]:
request_formatter_maps = (METHOD_NORMALIZERS, PYTHONIC_REQUEST_FORMATTERS,
ABI_REQUEST_FORMATTERS)
formatters = combine_formatters(request_formatter_maps, method_name)
return compose(*formatters)
def register(self, topic: Topic, enr: ENRAPI, current_time: float) -> None:
"""
Register a new ad.
A `ValueError` will be raised if the ad cannot be added because the table is full,
because the node already is present in the queue, or because the topic's wait time is
non-zero.
"""
queue = self.topic_queues[topic]
wait_time = self.get_wait_time(topic, current_time)
if wait_time > 0:
raise ValueError(
f"Topic queue or table is full (time to wait: {wait_time})")
present_node_ids = tuple(entry.node_id
for entry in self.get_enrs_for_topic(topic))
if enr.node_id in present_node_ids[:self.max_queue_size - 1]:
raise ValueError(
f"Topic queue already contains entry for node {encode_hex(enr.node_id)}"
)
if self.is_full:
queues = [queue for queue in self.topic_queues.values() if queue]
queue_with_oldest_ad = min(
queues,
key=toolz.compose(operator.attrgetter("registration_time"),
operator.itemgetter(-1)),
)
queue_with_oldest_ad.pop()
self.total_size -= 1
self.total_size -= len(queue)
queue.appendleft(Ad(enr=enr, registration_time=current_time))
self.total_size += len(queue)
class Status(BaseCommand[StatusPayload]):
protocol_command_id = 0
serialization_codec: RLPCodec[StatusPayload] = RLPCodec(
sedes=STATUS_STRUCTURE,
process_inbound_payload_fn=compose(
lambda args: StatusPayload(*args), ),
)
class ContractCodes(BaseCommand[ContractCodesPayload]):
protocol_command_id = 11
serialization_codec = RLPCodec(
sedes=CONTRACT_CODES_STRUCTURE,
process_inbound_payload_fn=compose(
lambda args: ContractCodesPayload(*args), ),
)
class ProofsV2(BaseCommand[ProofsPayloadV2]):
protocol_command_id = 16
serialization_codec = RLPCodec(
sedes=PROOFS_V2_STRUCTURE,
process_inbound_payload_fn=compose(
lambda args: ProofsPayloadV2(*args), ),
)
def get_error_formatters(method_name):
# Note error formatters work on the full response dict
# TODO - test this function
error_formatter_maps = ()
formatters = combine_formatters(error_formatter_maps, method_name)
return compose(*formatters)
def get_result_formatters(method_name):
formatters = combine_formatters((PYTHONIC_RESULT_FORMATTERS, ),
method_name)
attrdict_formatter = apply_formatter_if(is_dict and not_attrdict,
AttributeDict.recursive)
return compose(*formatters, attrdict_formatter)
def test_gap_continuity(changes):
MAX_BLOCK_NUM = 21
# method to get all the block numbers that are in a gap right now
_all_missing = compose(
set, partial(_all_gap_numbers, highest_block_number=MAX_BLOCK_NUM))
@to_set
def _all_inserted(chain_gaps):
"""List all the inserted headers, the block numbers not in gaps"""
missing = _all_missing(chain_gaps)
for block_num in range(MAX_BLOCK_NUM + 1):
if block_num not in missing:
yield block_num
chain_gaps = GENESIS_CHAIN_GAPS
for do_insert, block_num in changes:
starts_inserted = _all_inserted(chain_gaps)
starts_missing = _all_missing(chain_gaps)
if do_insert:
to_insert = block_num
_, chain_gaps = fill_gap(to_insert, chain_gaps)
assert not is_block_number_in_gap(to_insert, chain_gaps)
# Make sure that at most this one block number was filled
finished_inserted = _all_inserted(chain_gaps)
assert to_insert in finished_inserted
new_inserts = finished_inserted - starts_inserted
if block_num in starts_inserted:
assert new_inserts == set()
else:
assert new_inserts == {block_num}
# Make sure that no new gaps were created
finished_missing = _all_missing(chain_gaps)
assert to_insert not in finished_missing
new_missing = finished_missing - starts_missing
assert new_missing == set()
else:
to_remove = block_num
# Note that removing a header is inserting a gap
chain_gaps = reopen_gap(to_remove, chain_gaps)
assert is_block_number_in_gap(to_remove, chain_gaps)
# Make sure that no gaps were filled
finished_inserted = _all_inserted(chain_gaps)
new_inserts = finished_inserted - starts_inserted
assert new_inserts == set()
# Make sure that at most this one block number gap was reopened
finished_missing = _all_missing(chain_gaps)
new_missing = finished_missing - starts_missing
if block_num in starts_missing:
assert new_missing == set()
else:
assert new_missing == {block_num}
_validate_gap_invariants(chain_gaps)
def sign_and_send_raw_middleware(
make_request: Callable[[RPCEndpoint, Any], Any],
w3: "Web3") -> Callable[[RPCEndpoint, Any], RPCResponse]:
format_and_fill_tx = compose(format_transaction,
fill_transaction_defaults(w3),
fill_nonce(w3))
def middleware(method: RPCEndpoint, params: Any) -> RPCResponse:
if method != "eth_sendTransaction":
return make_request(method, params)
else:
transaction = format_and_fill_tx(params[0])
if 'from' not in transaction:
return make_request(method, params)
elif transaction.get('from') not in accounts:
return make_request(method, params)
account = accounts[transaction['from']]
raw_tx = account.sign_transaction(transaction).rawTransaction
return make_request(RPCEndpoint("eth_sendRawTransaction"),
[raw_tx])
return middleware
def get_error_formatters(
method_name: Union[RPCEndpoint, Callable[..., RPCEndpoint]]
) -> Callable[..., Any]:
# Note error formatters work on the full response dict
error_formatter_maps = (ERROR_FORMATTERS, )
formatters = combine_formatters(error_formatter_maps, method_name)
return compose(*formatters)
class Status(BaseCommand[StatusPayload]):
protocol_command_id = 0
serialization_codec: RLPCodec[StatusPayload] = RLPCodec(
sedes=sedes.List(
(sedes.big_endian_int, sedes.big_endian_int, sedes.big_endian_int,
hash_sedes, hash_sedes, ForkID)),
process_inbound_payload_fn=compose(
lambda args: StatusPayload(*args), ),
)
class NewBlock(BaseCommand[NewBlockPayload]):
protocol_command_id = 7
serialization_codec = RLPCodec(sedes=NEW_BLOCK_STRUCTURE,
process_inbound_payload_fn=compose(
lambda args: NewBlockPayload(*args),
apply_formatter_at_index(
lambda args: BlockFields(*args),
0,
)))
def get_error_formatters(
method_name: Union[RPCEndpoint, Callable[..., RPCEndpoint]]
) -> Dict[str, Callable[..., Any]]:
# Note error formatters work on the full response dict
# TODO - test this function
error_formatter_maps = ()
formatters = combine_formatters(error_formatter_maps, method_name)
return compose(*formatters)
class GetBlockHeaders(BaseCommand[GetBlockHeadersPayload]):
protocol_command_id = 2
serialization_codec = RLPCodec(
sedes=GET_BLOCK_HEADERS_STRUCTURE,
process_inbound_payload_fn=compose(
lambda args: GetBlockHeadersPayload(*args),
apply_formatter_at_index(lambda args: BlockHeadersQuery(*args),
1)),
)
def get_result_formatters(
method_name: Union[RPCEndpoint, Callable[..., RPCEndpoint]]
) -> Dict[str, Callable[..., Any]]:
formatters = combine_formatters((PYTHONIC_RESULT_FORMATTERS, ),
method_name)
attrdict_formatter = apply_formatter_if(is_dict and not_attrdict,
AttributeDict.recursive)
return compose(attrdict_formatter, *formatters)
def _find_oldest_unpruned_task_id(self, finished_task_id: TTaskID) -> TTaskID:
get_dependency_of_id = compose(
curry(do)(self._validate_has_task),
self._dependency_of,
attrgetter('task'),
self._tasks.get,
)
ancestors = iterate(get_dependency_of_id, finished_task_id)
return nth(self._max_depth, ancestors)
def normalize_result(msg: Tuple[BlockBody, ...]) -> BlockBodyBundles:
uncles_hashes = tuple(
map(compose(keccak, rlp.encode),
tuple(body.uncles for body in msg)))
transaction_roots_and_trie_data = tuple(
map(make_trie_root_and_nodes,
tuple(body.transactions for body in msg)))
body_bundles = tuple(
zip(msg, transaction_roots_and_trie_data, uncles_hashes))
return body_bundles
def _revert_pending_transaction_filter(self, filter):
is_valid_transaction_hash = excepts(
(TransactionNotFound,),
compose(
bool,
self.get_transaction_by_hash,
self.normalizer.normalize_outbound_transaction_hash,
),
lambda v: False,
)
values_to_remove = remove(is_valid_transaction_hash, filter.get_all())
filter.remove(*values_to_remove)
class NewBlock(BaseCommand[NewBlockPayload]):
protocol_command_id = 7
serialization_codec: RLPCodec[NewBlockPayload] = RLPCodec(
sedes=sedes.List((sedes.List(
(BlockHeader, sedes.CountableList(UninterpretedTransactionRLP),
sedes.CountableList(BlockHeader))), sedes.big_endian_int)),
process_inbound_payload_fn=compose(
lambda args: NewBlockPayload(*args),
apply_formatter_at_index(
lambda args: BlockFields(*args),
0,
)))
def _revert_block_filter(self, filter):
is_valid_block_hash = excepts(
(BlockNotFound,),
compose(
bool,
self.get_block_by_hash,
self.normalizer.normalize_outbound_block_hash,
),
lambda v: False,
)
values_to_remove = tuple(remove(is_valid_block_hash, filter.get_all()))
filter.remove(*values_to_remove)
class GetContractCodes(BaseCommand[GetContractCodesPayload]):
protocol_command_id = 10
serialization_codec = RLPCodec(
sedes=GET_CONTRACT_CODES_STRUCTURE,
process_inbound_payload_fn=compose(
lambda args: GetContractCodesPayload(*args),
apply_formatter_at_index(
apply_formatter_to_array(
lambda args: ContractCodeRequest(*args)),
1,
),
),
)
def _prune_forward(self, root_id: TTaskID, depth: int) -> Tuple[TTaskID]:
"""
Prune all forks forward from the root
"""
def prune_parent(prune_task_id: TTaskID) -> Set[TTaskID]:
children = self._dependencies.pop(prune_task_id, set())
del self._tasks[prune_task_id]
if prune_task_id in self._declared_finished:
self._declared_finished.remove(prune_task_id)
return children
prune_parent_list = compose(tuple, curry(mapcat)(prune_parent))
prune_trunk = repeat(prune_parent_list, depth)
return pipe((root_id, ), *prune_trunk)
def get_result_formatters(
method_name: Union[RPCEndpoint, Callable[..., RPCEndpoint]],
module: "Module",
) -> Dict[str, Callable[..., Any]]:
formatters = combine_formatters((PYTHONIC_RESULT_FORMATTERS, ),
method_name)
formatters_requiring_module = combine_formatters(
(FILTER_RESULT_FORMATTERS, ), method_name)
partial_formatters = apply_module_to_formatters(
formatters_requiring_module, module, method_name)
attrdict_formatter = apply_formatter_if(is_dict and not_attrdict,
AttributeDict.recursive)
return compose(*partial_formatters, attrdict_formatter, *formatters)
def get_request_formatters(
method_name: Union[RPCEndpoint, Callable[..., RPCEndpoint]]
) -> Dict[str, Callable[..., Any]]:
request_formatter_maps = (
ABI_REQUEST_FORMATTERS,
# METHOD_NORMALIZERS needs to be after ABI_REQUEST_FORMATTERS
# so that eth_getLogs's apply_formatter_at_index formatter
# is applied to the whole address
# rather than on the first byte of the address
METHOD_NORMALIZERS,
PYTHONIC_REQUEST_FORMATTERS,
)
formatters = combine_formatters(request_formatter_maps, method_name)
return compose(*formatters)
def mine_blocks(self, num_blocks=1, coinbase=None):
for _ in range(num_blocks):
block_to_mine = dissoc(self.block, 'hash')
block_hash = fake_rlp_hash(block_to_mine)
mined_block = assoc(block_to_mine, 'hash', block_hash)
assign_block_info = compose(
partial(assoc, key='block_number',
value=mined_block['number']),
partial(assoc, key='block_hash', value=mined_block['hash']),
)
mined_block['transactions'] = tuple(
assign_block_info(transaction)
for transaction in mined_block['transactions'])
self.blocks.append(mined_block)
self.block = make_block_from_parent(mined_block)
yield block_hash
def __init__(self, chain: BaseAsyncChain, peer_pool: BaseChainPeerPool,
stitcher: HeaderStitcher, token: CancelToken) -> None:
super().__init__(token=token)
self._chain = chain
self._stitcher = stitcher
max_pending_fillers = 50
self._filler_header_tasks = TaskQueue(
max_pending_fillers,
# order by block number of the parent header
compose(attrgetter('block_number'), itemgetter(0)),
)
# queue up idle peers, ordered by speed that they return block bodies
self._waiting_peers: WaitingPeers[TChainPeer] = WaitingPeers(
BaseBlockHeaders)
self._peer_pool = peer_pool
def _find_root(self, task_id: TTaskID) -> Tuple[TTaskID, int]:
"""
return the oldest root, and the depth to it from the seed task
"""
root_candidate = task_id
get_dependency_of_id = compose(self._dependency_of, attrgetter('task'), self._tasks.get)
# We'll use the maximum saved history (_max_depth) to cap how long the stale cache
# of history might get, when pruning. Increasing the cap should not be a problem, if needed.
for depth in range(0, self._max_depth):
dependency = get_dependency_of_id(root_candidate)
if dependency not in self._tasks:
return root_candidate, depth
else:
root_candidate = dependency
raise ValidationError(
f"Stale task history too long ({depth}) before pruning. {dependency} is still in cache."
)
def __init__(self, chain: AsyncChainAPI, peer_pool: BaseChainPeerPool,
stitcher: HeaderStitcher) -> None:
self.logger = get_logger('trinity.sync.common.headers.SkeletonSyncer')
self._chain = chain
self._stitcher = stitcher
max_pending_fillers = 50
self._filler_header_tasks = TaskQueue(
max_pending_fillers,
# order by block number of the parent header
compose(attrgetter('block_number'), itemgetter(0)),
)
# queue up idle peers, ordered by speed that they return block bodies
self._waiting_peers: WaitingPeers[TChainPeer] = WaitingPeers(
(ETHBlockHeaders, LESBlockHEaders), )
self._peer_pool = peer_pool
self.sync_progress: SyncProgress = None
def get_ancestors(self, limit: int, header: BlockHeaderAPI) -> Tuple[BlockAPI, ...]:
ancestor_count = min(header.block_number, limit)
# We construct a temporary block object
vm_class = self.get_vm_class_for_block_number(header.block_number)
block_class = vm_class.get_block_class()
block = block_class(header=header, uncles=[], transactions=[])
ancestor_generator = iterate(compose(
self.get_block_by_hash,
operator.attrgetter('parent_hash'),
operator.attrgetter('header'),
), block)
# we peel off the first element from the iterator which will be the
# temporary block object we constructed.
next(ancestor_generator)
return tuple(take(ancestor_count, ancestor_generator))
