async def test_blocks_by_root(enr, beacon_state):
async with SubprocessConn(cmd='rumor bare') as conn:
async with trio.open_nursery() as nursery:
rumor = Rumor(conn, nursery)
peer_id = await connect_rumor(rumor, enr)
req = List[Root, 64]([
'ef64a1b94652cd9070baa4f9c0e8b1ce624bdb071b77b51b1a54b8babb1a5cd2'
]).encode_bytes().hex()
blocks = []
async for chunk in rumor.rpc.blocks_by_root.req.raw(
peer_id, req, raw=True, compression='snappy').chunk():
if chunk['result_code'] == 0:
block = SignedBeaconBlock.decode_bytes(
bytes.fromhex(chunk['data']))
blocks.append(block)
compare_vals(1, len(blocks), 'num_blocks')
compare_containers(
SignedBeaconBlock(message=BeaconBlock(
state_root=
'0x363bcbffb8bcbc8db51bda09cb68a5a3cc7cb2c1b79f8301a3157e37e24c687d'
)), blocks[0])
nursery.cancel_scope.cancel()
async def run_work():
async with SubprocessConn(cmd='cd ../rumor && go run . bare') as conn:
# A Trio nursery hosts all the async tasks of the Rumor instance.
async with trio.open_nursery() as nursery:
# And optionally use Rumor(conn, debug=True) to be super verbose about Rumor communication.
await sync_experiment(Rumor(conn, nursery))
# Cancel the nursery to signal that we are not using Rumor anymore
nursery.cancel_scope.cancel()
async def wrapped_run_fn(args):
async with SubprocessConn(
cmd='./bin/rumor bare --level=trace') as conn:
async with trio.open_nursery() as nursery:
try:
rumor = Rumor(conn, nursery)
response_code = await async_run_fn(rumor, args)
return response_code
finally:
nursery.cancel_scope.cancel()
async def run_rumor_function(fn: Callable[[Rumor, trio.Nursery], Coroutine]):
async with trio.open_nursery() as nursery:
try:
async with SubprocessConn(
cmd='cd ../rumor && go run . bare') as conn:
# A Trio nursery hosts all the async tasks of the Rumor instance.
async with trio.open_nursery() as nursery:
# And optionally use Rumor(conn, debug=True) to be super verbose about Rumor communication.
await fn(Rumor(conn, nursery), nursery)
# Cancel the nursery to signal that we are not using Rumor anymore
nursery.cancel_scope.cancel()
except Exception as e:
print(e)
async def test_teku():
config = InstanceConfig('teku', BEACON_STATE_LOCATION, TEST_ENR)
start_instance(config)
try:
async with SubprocessConn(
cmd='./bin/rumor bare --level=trace') as conn:
async with trio.open_nursery() as nursery:
rumor = Rumor(conn, nursery)
peer_id = await connect_rumor(rumor, 'teku', TEST_ENR)
# NOTE: this may not be the exact test
assert peer_id is not None
nursery.cancel_scope.cancel()
finally:
stop_instance(config.client)
async def server_blocks_by_range_example(rumor: Rumor, nursery: trio.Nursery):
# Morty is us
morty = rumor.actor('morty')
await morty.host.start()
await morty.host.listen(tcp=9000)
print("started morty")
# Rick is the other client
rick_enr = "enr:-Iu4QGuiaVXBEoi4kcLbsoPYX7GTK9ExOODTuqYBp9CyHN_PSDtnLMCIL91ydxUDRPZ-jem-o0WotK6JoZjPQWhTfEsTgmlkgnY0gmlwhDbOLfeJc2VjcDI1NmsxoQLVqNEoCVTC74VmUx25USyFe7lL0TgpXHaCX9CDy9H6boN0Y3CCIyiDdWRwgiMo"
rick_peer_id = await morty.peer.connect(rick_enr, "bootnode").peer_id()
print(f"connected to Rick {rick_peer_id}")
call = morty.rpc.blocks_by_range.listen(raw=True, compression='snappy')
print("listening for requests")
async for req in call.req():
print(f"morty: Got request: {req}")
parsed_req = BlocksByRange.decode_bytes(
bytes.fromhex(req['chunk']['data']))
print('parsed request: ', parsed_req)
start = parsed_req.start_slot
end = start + parsed_req.count * parsed_req.step
for i, slot in zip(range(parsed_req.count),
range(start, end, parsed_req.step)):
# Try any message:
# resp = f"not a block, but can you decode this chunk though? chunk nr {i} here".encode()
# Or construct a block (can make it more consensus-valid, but snappy compression testing can be simple):
resp = SignedBeaconBlock(message=BeaconBlock(
slot=slot)).encode_bytes().hex()
print(f"responding chunk {i} slot {slot} chunk: {resp}")
await morty.rpc.blocks_by_range.resp.chunk.raw(
req['req_id'], resp, done=(i + 1 == parsed_req.count))
print("done responding")
print("morty: stopped listening for requests")
async def check_topics(enr):
async with SubprocessConn(cmd='rumor bare') as conn:
async with trio.open_nursery() as nursery:
rumor = Rumor(conn, nursery)
peer_id = await connect_rumor(rumor, enr)
await rumor.gossip.start()
peer_id = await rumor.peer.connect(enr).peer_id()
for topic in EXPECTED_TOPICS:
await rumor.gossip.join(topic)
resp = await rumor.gossip.list_peers(topic)
peers = resp['peers']
assert peers == [peer_id], f'actual peers for topic {topic}: {peers}'
nursery.cancel_scope.cancel()
async def server_blocks_by_root_example(rumor: Rumor, nursery: trio.Nursery):
# Morty is us
morty = rumor.actor('morty')
await morty.host.start()
await morty.host.listen(tcp=9000)
print("started morty")
# Rick is the other client
rick_enr = "enr:-Iu4QGuiaVXBEoi4kcLbsoPYX7GTK9ExOODTuqYBp9CyHN_PSDtnLMCIL91ydxUDRPZ-jem-o0WotK6JoZjPQWhTfEsTgmlkgnY0gmlwhDbOLfeJc2VjcDI1NmsxoQLVqNEoCVTC74VmUx25USyFe7lL0TgpXHaCX9CDy9H6boN0Y3CCIyiDdWRwgiMo"
rick_peer_id = await morty.peer.connect(rick_enr, "bootnode").peer_id()
print(f"connected to Rick {rick_peer_id}")
call = morty.rpc.blocks_by_root.listen(raw=True, compression='snappy')
print("listening for requests")
async for req in call.req():
print(f"morty: Got request: {req}")
parsed_req = BlocksByRoot.decode_bytes(
bytes.fromhex(req['chunk']['data']))
print('parsed request: ', parsed_req)
for i, root in enumerate(parsed_req):
resp = SignedBeaconBlock(message=BeaconBlock(
slot=slot)).encode_bytes().hex()
print(f"responding chunk {i} root {root}, chunk: {resp}")
await morty.rpc.blocks_by_range.resp.chunk.raw(
req['req_id'], resp, done=(i + 1 == len(parsed_req)))
print("done responding")
print("morty: stopped listening for requests")
async def basic_status_example(rumor: Rumor, nursery: trio.Nursery):
# Load some genesis state of the client (or use make_genesis.py)
state = load_state('genesis.ssz')
# Morty is us
morty = rumor.actor('morty')
await morty.host.start()
await morty.host.listen(tcp=9000)
print("started morty")
# Rick is the other client
rick_enr = "enr:-Iu4QGuiaVXBEoi4kcLbsoPYX7GTK9ExOODTuqYBp9CyHN_PSDtnLMCIL91ydxUDRPZ-jem-o0WotK6JoZjPQWhTfEsTgmlkgnY0gmlwhDbOLfeJc2VjcDI1NmsxoQLVqNEoCVTC74VmUx25USyFe7lL0TgpXHaCX9CDy9H6boN0Y3CCIyiDdWRwgiMo"
rick_peer_id = await morty.peer.connect(rick_enr, "bootnode").peer_id()
print(f"connected to Rick {rick_peer_id}")
print("Testing a Status RPC request")
head = state.latest_block_header.copy()
head.state_root = state.hash_tree_root()
# Sync status
morty_status = Status(
version=compute_fork_digest(state.fork.current_version,
state.genesis_validators_root),
finalized_root=state.finalized_checkpoint.root,
finalized_epoch=0,
head_root=head.hash_tree_root(),
head_epoch=0,
)
req = morty_status.encode_bytes().hex()
print(f"morty: sending rick a status request: {req}")
# Note: public testnet node is not updated, only receiving an empty response if snappy is enabled.
resp = await morty.rpc.status.req.raw(rick_peer_id, req, raw=True)
print(f"morty: received status response from rick: {resp}")
try:
rick_status = Status.decode_bytes(bytes.fromhex(resp['chunk']['data']))
print(rick_status)
except Exception as e:
print(f"could not decode status response: {e}")
call = morty.rpc.status.listen(raw=True, compression='snappy')
# Other keywords to try here:
# Req-resp timeout: timeout=123000 (in milliseconds, 0 to disable)
# Drop contents, not keeping track of them to reply later: drop=True
# Ignore request bytes, do not read any: read=False
async def process_requests():
async for req in call.req():
print(f"morty: Got request: {req}")
# Respond with Input error
# await morty.rpc.status.resp.invalid_request(req['req_id'], f"hello! Morty does not like your request!")
# Respond with server error
# await morty.rpc.status.resp.server_error(req['req_id'], f"hello! Morty failed, look for a new morty!")
# Respond with valid chunk (and done=True to exit immediately after)
resp = morty_status.encode_bytes().hex()
await morty.rpc.status.resp.chunk.raw(req['req_id'],
resp,
done=True)
# Or send arbitrary data
# resp = bytes.fromhex('1337')
# await morty.rpc.status.resp.chunk.raw(req['req_id'], resp, result_code=2, done=True)
print("morty: stopped listening for requests")
print("listening for requests")
await process_requests()
async def lit_morty(rumor: Rumor):
state = load_state('lighthouse/genesis.ssz')
morty = rumor.actor('morty')
await morty.host.start()
await morty.host.listen(tcp=9000)
genesis_root = state.hash_tree_root()
# Sync status
morty_status = Status(
version=spec.GENESIS_FORK_VERSION,
finalized_root=genesis_root,
finalized_epoch=0,
head_root=genesis_root,
head_epoch=0,
)
print("Morty status:")
print(morty_status)
print(morty_status.encode_bytes().hex())
async def sync_from_bootnode(state: spec.BeaconState):
await trio.sleep(2)
peer_id = await morty.peer.connect(bootnodes[0], "bootnode").peer_id()
print(f"connected bootnode peer: {peer_id}")
# boot_status = await ask_status(peer_id)
# print("Bootnode status:", boot_status)
print(await morty.peer.list('all'))
# Play nice, don't hit them with another request right away, wait half a minute. (Age?!)
# await trio.sleep(31)
async def sync_step(stats_csv: csv.DictWriter,
epochs_ctx: fast_spec.EpochsContext,
state: fast_spec.BeaconState) -> spec.BeaconState:
range_req = BlocksByRange(start_slot=state.slot + 1,
count=20,
step=1).encode_bytes().hex()
print("range req:", range_req)
blocks = []
async for chunk in morty.rpc.blocks_by_range.req.raw(
peer_id, range_req, max_chunks=10, raw=True).chunk():
print("got chunk: ", chunk)
if chunk['result_code'] == 0:
block = spec.SignedBeaconBlock.decode_bytes(
bytes.fromhex(chunk["data"]))
blocks.append(block)
else:
print("failed to get block; msg: ", chunk["msg"])
break
for b in blocks:
print("processing block!")
start_time = time.time()
if state.slot > 500 and (state.slot +
1) % fast_spec.SLOTS_PER_EPOCH == 0:
with io.open('pre.ssz', 'bw') as f:
state.serialize(f)
with io.open('block.ssz', 'bw') as f:
b.serialize(f)
import sys
sys.exit(1)
transition_input_state = state.copy()
fast_spec.state_transition(epochs_ctx, transition_input_state,
b)
end_time = time.time()
elapsed_time = end_time - start_time
print(
f"slot: {state.slot} state root: {state.hash_tree_root().hex()} processing speed: {1.0 / elapsed_time} blocks / second ({elapsed_time * 1000.0} ms/block)"
)
def subtree_size(n: Node) -> int:
if n.is_leaf():
return 1
return subtree_size(n.get_left()) + subtree_size(
n.get_right())
def analyze_diff(a: Node, b: Node) -> (int, int):
"""Iterate over the changes of b, not common with a. Left-to-right order.
Returns (a,b) tuples that can't be diffed deeper."""
if a.root != b.root:
a_leaf = a.is_leaf()
b_leaf = b.is_leaf()
if a_leaf or b_leaf:
return subtree_size(a), subtree_size(b)
else:
a_l, b_l = analyze_diff(a.get_left(), b.get_left())
a_r, b_r = analyze_diff(a.get_right(),
b.get_right())
return a_l + a_r + 1, b_l + b_r + 1
return 0, 0
stats = {
'slot': b.message.slot,
'proposer': epochs_ctx.get_beacon_proposer(b.message.slot),
'process_time': elapsed_time,
}
# for i, key in enumerate(spec.BeaconState.fields().keys()):
# a = state.get(i).get_backing()
# b = transition_input_state.get(i).get_backing()
#
# removed_nodes, added_nodes = analyze_diff(a, b)
# stats[f"added_nodes_{key}"] = added_nodes
# stats[f"removed_nodes_{key}"] = removed_nodes
# stats_csv.writerow(stats)
print(f"stats: {stats}")
state = transition_input_state
global morty_status
morty_status = Status(
version=spec.GENESIS_FORK_VERSION,
finalized_root=state.finalized_checkpoint.root,
finalized_epoch=state.finalized_checkpoint.epoch,
head_root=state.latest_block_header.hash_tree_root(),
head_epoch=state.slot,
)
return state
async def sync_work(stats_csv: csv.DictWriter,
state: spec.BeaconState):
epochs_ctx = fast_spec.EpochsContext()
epochs_ctx.load_state(state)
while True:
state = await sync_step(stats_csv, epochs_ctx, state)
if state.slot > 10000:
break # synced enough (TODO: use bootnode status instead)
with open(r'sync_stats.csv', 'a', newline='') as csvfile:
fieldnames = ['slot', 'proposer', 'process_time']
for key in spec.BeaconState.fields().keys():
fieldnames.append(f"added_nodes_{key}")
fieldnames.append(f"removed_nodes_{key}")
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
await sync_work(writer, state)
print("Saying goobye")
ok_bye_bye = Goodbye(1) # A.k.a "Client shut down"
await morty.rpc.goodbye.req.raw(peer_id,
ok_bye_bye.encode_bytes().hex(),
raw=True)
print("disconnecting")
await morty.peer.disconnect(peer_id)
print("disconnected")
await sync_from_bootnode(state)
# Close the Rumor process
await rumor.stop()
async def run_work():
# Hook it up to your own local version of Rumor, if you like.
# And optionally enable debug=True to be super verbose about Rumor communication.
async with Rumor(cmd='cd ../rumor && go run .') as rumor:
await lit_morty(rumor)
async def sync_experiment(rumor: Rumor):
state = load_state('genesis.ssz')
morty = rumor.actor('morty')
await morty.host.start()
await morty.host.listen(tcp=9000)
head = state.latest_block_header.copy()
head.state_root = state.hash_tree_root()
# Sync status
morty_status = Status(
version=compute_fork_digest(state.fork.current_version,
state.genesis_validators_root),
finalized_root=state.finalized_checkpoint.root,
finalized_epoch=0,
head_root=head.hash_tree_root(),
head_epoch=0,
)
print("Morty status:")
print(morty_status)
print(morty_status.encode_bytes().hex())
async def sync_work():
await trio.sleep(2)
rick_enr = "enr:-Iu4QGuiaVXBEoi4kcLbsoPYX7GTK9ExOODTuqYBp9CyHN_PSDtnLMCIL91ydxUDRPZ-jem-o0WotK6JoZjPQWhTfEsTgmlkgnY0gmlwhDbOLfeJc2VjcDI1NmsxoQLVqNEoCVTC74VmUx25USyFe7lL0TgpXHaCX9CDy9H6boN0Y3CCIyiDdWRwgiMo"
peer_id = await morty.peer.connect(rick_enr, "bootnode").peer_id()
print(f"connected bootnode peer: {peer_id}")
print(await morty.peer.list('all'))
async def sync_step(current_slot: Slot):
slot_count = 20
range_req = BlocksByRange(start_slot=current_slot + 1,
count=slot_count,
step=1).encode_bytes().hex()
print("range req:", range_req)
blocks = []
async for chunk in morty.rpc.blocks_by_range.req.raw(
peer_id, range_req, max_chunks=slot_count,
raw=True).chunk():
print("got chunk: ", chunk)
if chunk['result_code'] == 0:
block = SignedBeaconBlock.decode_bytes(
bytes.fromhex(chunk["data"]))
blocks.append(block)
current_slot = block.message.slot
else:
print("failed to get block; msg: ", chunk["msg"])
break
# TODO: could process blocks using as done with previous sync experiment. Just need to test RPC now though.
return current_slot
async def sync_work():
current_slot = Slot(0)
while True:
current_slot = await sync_step(current_slot)
if current_slot > 100:
break
await sync_work()
print("Saying goobye")
ok_bye_bye = Goodbye(1) # A.k.a "Client shut down"
await morty.rpc.goodbye.req.raw(peer_id,
ok_bye_bye.encode_bytes().hex(),
raw=True)
print("disconnecting")
await morty.peer.disconnect(peer_id)
print("disconnected")
await sync_work()