def generate_genesis_state(cls, get_genesis_time: Callable[[], Timestamp],
network_dir: Path, keymap: Dict[Any, Any],
clients: Tuple[Client, ...]) -> None:
logger = cls.get_logger()
state_machine_class = XiaoLongBaoStateMachine
# Since create_mock_genesis takes a long time, update the real genesis_time later
dummy_time = Timestamp(int(time.time()))
state, _ = create_mock_genesis(
num_validators=len(keymap.keys()),
config=state_machine_class.config,
keymap=keymap,
genesis_block_class=state_machine_class.block_class,
genesis_time=dummy_time,
)
genesis_time = get_genesis_time()
logger.info(
"Genesis time will be %s from now",
humanize_seconds(genesis_time - int(time.time())),
)
state = state.copy(genesis_time=genesis_time, )
# The output here can be trusted, so use unsafe mode for performance
yaml = YAML(typ='unsafe')
with open(network_dir / GENESIS_FILE, "w") as f:
yaml.dump(to_formatted_dict(state), f)
# Distribute genesis file to clients
for client in clients:
with open(client.client_dir / GENESIS_FILE, "w") as f:
yaml.dump(to_formatted_dict(state), f)
def generate_genesis_state(cls, get_genesis_time: Callable[[], Timestamp],
network_dir: Path, keymap: Dict[BLSPubkey, int],
clients: Tuple[Client, ...]) -> None:
logger = cls.get_logger()
state_machine_class = XiaoLongBaoStateMachine
# NOTE: see https://github.com/ethereum/trinity/issues/786
override_lengths(XiaoLongBaoStateMachine.config)
# Since create_mock_genesis takes a long time, update the real genesis_time later
dummy_time = Timestamp(int(time.time()))
state, _ = create_mock_genesis(
pubkeys=cast(
Sequence[BLSPubkey],
keymap.keys(),
),
config=state_machine_class.config,
keymap=keymap,
genesis_block_class=state_machine_class.block_class,
genesis_time=dummy_time,
)
genesis_time = get_genesis_time()
logger.info(
"Genesis time will be %s from now",
humanize_seconds(genesis_time - int(time.time())),
)
state = state.copy(genesis_time=genesis_time, )
# The output here can be trusted, so use unsafe mode for performance
yaml = YAML(typ='unsafe')
with open(network_dir / GENESIS_FILE, "w") as f:
yaml.dump(to_formatted_dict(state), f)
# Distribute genesis file to clients
for client in clients:
with open(client.client_dir / GENESIS_FILE, "w") as f:
yaml.dump(to_formatted_dict(state), f)
def generate_genesis_state(cls, genesis_delay: Second, network_dir: Path,
keymap: Dict[Any, Any],
clients: Tuple[Client, ...]) -> None:
logger = cls.get_logger()
state_machine_class = XiaoLongBaoStateMachine
# Since create_mock_genesis takes a long time, update the real genesis_time later
dummy_time = Timestamp(int(time.time()))
state, _ = create_mock_genesis(
num_validators=len(keymap.keys()),
config=state_machine_class.config,
keymap=keymap,
genesis_block_class=state_machine_class.block_class,
genesis_time=dummy_time,
)
logger.info(f"Genesis time will be {genesis_delay} seconds from now")
genesis_time = Timestamp(int(time.time()) + genesis_delay)
state = state.copy(genesis_time=genesis_time, )
yaml = YAML()
with open(network_dir / GENESIS_FILE, "w") as f:
yaml.dump(to_formatted_dict(state), f)
# Distribute genesis file to clients
for client in clients:
with open(client.client_dir / GENESIS_FILE, "w") as f:
yaml.dump(to_formatted_dict(state), f)
def validate_state(post_state: BeaconState,
expected_state: BeaconState) -> None:
# Use dict diff, easier to see the diff
dict_post_state = to_formatted_dict(post_state, BeaconState)
dict_expected_state = to_formatted_dict(expected_state, BeaconState)
for key, value in dict_expected_state.items():
if isinstance(value, list):
value = tuple(value)
if dict_post_state[key] != value:
raise AssertionError(f"state.{key} is incorrect:\n"
f"\tExpected: {value}\n"
f"\tResult: {dict_post_state[key]}\n")
def test_not_serializable():
octopi = (octopus, octopus, octopus)
sedes = List(Animal, 2**32)
output = to_formatted_dict(octopi, sedes)
hashable_octopi = HashableList.from_iterable(octopi, sedes)
assert hashable_octopi == from_formatted_dict(output, List(Animal, 2**32))
async def getCanonicalBlockBySlot(self, slot: Slot) -> Dict[Any, Any]:
"""
Return the canonical block of the given slot.
"""
block = await self.chain.coro_get_canonical_block_by_slot(
slot, SignedBeaconBlock)
return to_formatted_dict(block, sedes=SignedBeaconBlock)
async def getBlockByRoot(self, root: Root) -> Dict[Any, Any]:
"""
Return the block of given root.
"""
block = await self.chain.coro_get_block_by_root(
root, SignedBeaconBlock)
return to_formatted_dict(block, sedes=SignedBeaconBlock)
async def block(self, request: web.Request) -> Dict[str, Any]:
if 'slot' in request.query:
slot = Slot(int(request.query['slot']))
block = self.chain.get_canonical_block_by_slot(slot)
elif 'root' in request.query:
root = cast(Root, decode_hex(request.query['root']))
block = self.chain.get_block_by_root(root)
return to_formatted_dict(block, sedes=BeaconBlock)
async def state(self, request: web.Request) -> Dict[str, Any]:
if 'slot' in request.query:
slot = Slot(int(request.query['slot']))
state = self.chain.get_state_by_slot(slot)
elif 'root' in request.query:
root = cast(Root, decode_hex(request.query['root']))
state = self.chain.get_state_by_root(root)
else:
raise APIServerError(f"Wrong querystring: {request.query}")
return to_formatted_dict(state)
def test_custom_codec():
class CustomCodec(DefaultCodec):
@staticmethod
def format_integer(value, sedes):
return encode_hex(sedes.serialize(value))
@staticmethod
def unformat_integer(value, sedes):
return sedes.deserialize(decode_hex(value))
output = to_formatted_dict(zoo, codec=CustomCodec)
read_zoo = from_formatted_dict(output, Zoo, CustomCodec)
assert read_zoo == zoo
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-ssz", type=str, required=True)
parser.add_argument("-config", type=str, required=True)
args = parser.parse_args()
config_path = Path(args.config)
minimal_config = load_config_at_path(config_path)
override_lengths(minimal_config)
with open(args.ssz, "rb") as f:
encoded = f.read()
state = ssz.decode(encoded, sedes=BeaconState)
yaml = YAML(typ="unsafe")
yaml.dump(to_formatted_dict(state), sys.stdout)
def execute_state_transtion(test_case, base_db):
bls.use_noop_backend()
dict_config = test_case['config']
verify_signatures = test_case['verify_signatures']
dict_initial_state = test_case['initial_state']
dict_blocks = test_case['blocks']
dict_expected_state = test_case['expected_state']
# TODO: make it case by case
assert verify_signatures is False
# Set config
config = generate_config_by_dict(dict_config)
# Set Vector fields
override_vector_lengths(config)
# Set pre_state
pre_state = from_formatted_dict(dict_initial_state, BeaconState)
# Set blocks
blocks = ()
for dict_block in dict_blocks:
block = from_formatted_dict(dict_block, SerenityBeaconBlock)
blocks += (block, )
sm_class = SerenityStateMachine.configure(
__name__='SerenityStateMachineForTesting',
config=config,
)
chaindb = BeaconChainDB(base_db, Eth2GenesisConfig(config))
attestation_pool = AttestationPool()
post_state = pre_state.copy()
for block in blocks:
sm = sm_class(chaindb, attestation_pool, None, post_state)
post_state, _ = sm.import_block(block)
# Use dict diff, easier to see the diff
dict_post_state = to_formatted_dict(post_state, BeaconState)
for key, value in dict_expected_state.items():
if isinstance(value, list):
value = tuple(value)
assert dict_post_state[key] == value
async def _handle_goodbye(self, stream: INetStream) -> None:
peer_id = stream.mplex_conn.peer_id
self.logger.debug("Waiting for goodbye from %s", peer_id)
try:
goodbye = await read_req(stream, Goodbye)
has_error = False
except (ReadMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
has_error = True
if isinstance(error, ReadMessageFailure):
await stream.reset()
elif isinstance(error, MplexStreamEOF):
await stream.close()
self.logger.debug("Received the goodbye message %s",
to_formatted_dict(goodbye))
if not has_error:
await stream.close()
await self.disconnect_peer(peer_id)
def test_dump_serializble_with_explicit_sedes():
to_formatted_dict(zoo, Zoo)
async def say_hello(self, peer_id: ID) -> None:
self.logger.info("Say hello to %s", str(peer_id))
hello_mine = self._make_hello_packet()
self.logger.debug(
"Opening new stream to peer=%s with protocols=%s",
peer_id,
[REQ_RESP_HELLO_SSZ],
)
stream = await self.host.new_stream(peer_id, [REQ_RESP_HELLO_SSZ])
self.logger.debug("Sending our hello message %s",
to_formatted_dict(hello_mine))
try:
await write_req(stream, hello_mine)
has_error = False
except (WriteMessageFailure, MplexStreamEOF,
MplexStreamReset) as error:
has_error = True
if isinstance(error, WriteMessageFailure):
await stream.reset()
elif isinstance(error, MplexStreamEOF):
await stream.close()
finally:
if has_error:
await self.disconnect_peer(peer_id)
error_msg = f"fail to write request={hello_mine}"
self.logger.info("Handshake failed: %s", error_msg)
raise HandshakeFailure(error_msg)
self.logger.debug("Waiting for hello from the other side")
try:
resp_code, hello_other_side = await read_resp(stream, HelloRequest)
has_error = False
except (ReadMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
has_error = True
if isinstance(error, ReadMessageFailure):
await stream.reset()
elif isinstance(error, MplexStreamEOF):
await stream.close()
finally:
if has_error:
await self.disconnect_peer(peer_id)
self.logger.info("Handshake failed: fail to read the response")
raise HandshakeFailure("fail to read the response")
self.logger.debug(
"Received the hello message %s, resp_code=%s",
to_formatted_dict(hello_other_side),
resp_code,
)
# TODO: Handle the case when `resp_code` is not success.
if resp_code != ResponseCode.SUCCESS:
# TODO: Do something according to the `ResponseCode`
error_msg = (
"resp_code != ResponseCode.SUCCESS, "
f"resp_code={resp_code}, error_msg={to_formatted_dict(hello_other_side)}"
)
self.logger.info("Handshake failed: %s", error_msg)
await stream.reset()
await self.disconnect_peer(peer_id)
raise HandshakeFailure(error_msg)
hello_other_side = cast(HelloRequest, hello_other_side)
try:
await self._validate_hello_req(hello_other_side)
except ValidationError as error:
error_msg = (
f"hello message {to_formatted_dict(hello_other_side)} is invalid: {str(error)}"
)
self.logger.info(
"Handshake failed: %s. Disconnecting %s",
error_msg,
peer_id,
)
await stream.reset()
await self.say_goodbye(peer_id,
GoodbyeReasonCode.IRRELEVANT_NETWORK)
await self.disconnect_peer(peer_id)
raise HandshakeFailure(error_msg) from error
if peer_id not in self.handshaked_peers:
peer = Peer.from_hello_request(self, peer_id, hello_other_side)
self.handshaked_peers.add(peer)
self.logger.debug(
"Handshake to peer=%s is finished. Added to the `handshake_peers`",
peer_id,
)
# Check if we are behind the peer
self._compare_chain_tip_and_finalized_epoch(
hello_other_side.finalized_epoch,
hello_other_side.head_slot,
)
await stream.close()
async def getFinalizedHead(self) -> Dict[Any, Any]:
"""
Return finalized head block.
"""
block = await self.chain.coro_get_finalized_head(SignedBeaconBlock)
return to_formatted_dict(block, sedes=SignedBeaconBlock)
def munge_all_args(cls, args: Namespace, trinity_config: TrinityConfig) -> None:
logger = cls.get_logger()
logger.info("Configuring testnet")
override_lengths(MINIMAL_SERENITY_CONFIG)
if args.wipedb:
beacon_config = trinity_config.get_app_config(BeaconAppConfig)
logger.info(f'Blowing away the database: {beacon_config.database_dir}')
try:
shutil.rmtree(beacon_config.database_dir)
except FileNotFoundError:
# there's nothing to wipe, that's fine!
pass
else:
beacon_config.database_dir.mkdir()
genesis_path = args.genesis_state_ssz_path or Path('resources/genesis.ssz')
logger.info(f"Using genesis from {genesis_path}")
# read the genesis!
try:
with open(genesis_path, 'rb') as f: # type: IO[Any]
encoded = f.read()
state = ssz.decode(encoded, sedes=BeaconState)
except FileNotFoundError:
import os
logger.critical(
"Required: the genesis state at %s/genesis.ssz,"
" or the path to this state with command line"
" argument `--genesis-state-ssz-path $PATH`",
os.getcwd(),
)
sys.exit(1)
now = int(time.time())
if args.start_time:
if args.start_time <= now:
logger.warning(f"--start-time must be a time in the future. Current time is {now}")
delta = args.start_time - now
logger.info("Time will begin %d seconds from now", delta)
# adapt the state, then print the new root!
state = state.copy(
genesis_time=args.start_time
)
elif args.start_delay:
if args.start_delay < 0:
logger.info(f"--start-time must be positive")
sys.exit(1)
start_time = now + args.start_delay
logger.info("Genesis time is %d", start_time)
state = state.copy(
genesis_time=start_time
)
else:
logger.info("Using genesis_time from genesis state to determine start time")
logger.info(f"Genesis hash tree root: {state.hash_tree_root.hex()}")
logger.info(f"Configuring {trinity_config.trinity_root_dir}")
# Save the genesis state to the data dir!
yaml = YAML(typ='unsafe')
with open(trinity_config.trinity_root_dir / GENESIS_FILE, 'w') as f:
yaml.dump(to_formatted_dict(state), f)
# Save the validator keys to the data dir
keys_dir = trinity_config.trinity_root_dir / VALIDATOR_KEY_DIR
try:
shutil.rmtree(keys_dir)
except FileNotFoundError:
pass
keys_dir.mkdir()
def parse_key(maybe_hexstr: str) -> Union[int, str]:
try:
return to_int(hexstr=maybe_hexstr)
except TypeError:
return maybe_hexstr
validators = args.validators
if (args.validators or args.validators_from_yaml_key_file):
if not validators:
validators_keys_file = args.validators_from_yaml_key_file
yaml = YAML(typ="unsafe")
keys = yaml.load(Path(validators_keys_file))
for (i, key) in enumerate(keys):
file_name = f"v_{i}.privkey"
key_path = keys_dir / file_name
with open(key_path, "w") as f:
f.write(str(parse_key(key['privkey'])))
else:
validators = [int(token) for token in validators.split(',')]
for validator in validators:
if validator < 0 or validator > 15:
logger.error(f"{validator} is not a valid validator")
sys.exit(1)
logger.info(f"Validating: {validators}")
yaml = YAML(typ="unsafe")
keys = yaml.load(
Path(
'eth2/beacon/scripts/quickstart_state/keygen_16_validators.yaml'
)
)
for validator_index in validators:
key = keys[validator_index]
file_name = f"v{validator_index:07d}.privkey"
key_path = keys_dir / file_name
with open(key_path, "w") as f:
f.write(str(parse_key(key['privkey'])))
else:
logger.info("not running any validators")
# disable some components which shouldn't be running
args.disable_discovery = True
args.disable_upnp = True
args.network_tracking_backend = TrackingBackend.do_not_track
async def _handle_beacon_blocks(self, stream: INetStream) -> None:
peer_id = stream.mplex_conn.peer_id
if peer_id not in self.handshaked_peers:
self.logger.info(
"Processing beacon blocks request failed: not handshaked with peer=%s yet",
peer_id,
)
await stream.reset()
return
self.logger.debug(
"Waiting for beacon blocks request from the other side")
try:
beacon_blocks_request = await read_req(stream, BeaconBlocksRequest)
has_error = False
except (ReadMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
has_error = True
if isinstance(error, ReadMessageFailure):
await stream.reset()
elif isinstance(error, MplexStreamEOF):
await stream.close()
finally:
if has_error:
return
self.logger.debug("Received the beacon blocks request message %s",
to_formatted_dict(beacon_blocks_request))
try:
requested_head_block = self.chain.get_block_by_hash_tree_root(
beacon_blocks_request.head_block_root)
except (BlockNotFound, ValidationError) as error:
self.logger.info("Sending empty blocks, reason: %s", error)
# We don't have the chain data peer is requesting
requested_beacon_blocks: Tuple[BaseBeaconBlock, ...] = tuple()
else:
# Check if slot of specified head block is greater than specified start slot
if requested_head_block.slot < beacon_blocks_request.start_slot:
reason = (
f"Invalid request: head block slot({requested_head_block.slot})"
f" lower than `start_slot`({beacon_blocks_request.start_slot})"
)
try:
await write_resp(stream, reason,
ResponseCode.INVALID_REQUEST)
has_error = False
except (WriteMessageFailure, MplexStreamEOF,
MplexStreamReset) as error:
has_error = True
if isinstance(error, WriteMessageFailure):
await stream.reset()
elif isinstance(error, MplexStreamEOF):
await stream.close()
finally:
if has_error:
self.logger.info(
"Processing beacon blocks request failed: failed to write message %s",
reason,
)
return
await stream.close()
return
else:
try:
requested_beacon_blocks = self._get_requested_beacon_blocks(
beacon_blocks_request, requested_head_block)
except ValidationError as val_error:
reason = "Invalid request: " + str(val_error)
try:
await write_resp(stream, reason,
ResponseCode.INVALID_REQUEST)
has_error = False
except (WriteMessageFailure, MplexStreamEOF,
MplexStreamReset) as error:
has_error = True
if isinstance(error, WriteMessageFailure):
await stream.reset()
elif isinstance(error, MplexStreamEOF):
await stream.close()
finally:
if has_error:
self.logger.info(
"Processing beacon blocks request failed: "
"failed to write message %s",
reason,
)
return
await stream.close()
return
# TODO: Should it be a successful response if peer is requesting
# blocks on a fork we don't have data for?
beacon_blocks_response = BeaconBlocksResponse(
blocks=requested_beacon_blocks)
self.logger.debug("Sending beacon blocks response %s",
beacon_blocks_response)
try:
await write_resp(stream, beacon_blocks_response,
ResponseCode.SUCCESS)
has_error = False
except (WriteMessageFailure, MplexStreamEOF,
MplexStreamReset) as error:
has_error = True
if isinstance(error, WriteMessageFailure):
await stream.reset()
elif isinstance(error, MplexStreamEOF):
await stream.close()
finally:
if has_error:
self.logger.info(
"Processing beacon blocks request failed: failed to write message %s",
beacon_blocks_response,
)
return
self.logger.debug(
"Processing beacon blocks request from %s is finished",
peer_id,
)
await stream.close()
async def head(self, request: web.Request) -> Dict[str, Any]:
return to_formatted_dict(self.chain.get_canonical_head(),
sedes=BeaconBlock)
def test_parsing_and_dumping():
json_str = json.dumps(to_formatted_dict(zoo))
read_zoo = from_formatted_dict(json.loads(json_str), Zoo)
assert read_zoo == zoo
def test_not_serializable():
octopi = (octopus, octopus, octopus)
output = to_formatted_dict(octopi, List(Animal))
assert octopi == from_formatted_dict(output, List(Animal))
async def _handle_hello(self, stream: INetStream) -> None:
# TODO: Find out when we should respond the `ResponseCode`
# other than `ResponseCode.SUCCESS`.
peer_id = stream.mplex_conn.peer_id
self.logger.debug("Waiting for hello from the other side")
try:
hello_other_side = await read_req(stream, HelloRequest)
has_error = False
except (ReadMessageFailure, MplexStreamEOF, MplexStreamReset) as error:
has_error = True
if isinstance(error, ReadMessageFailure):
await stream.reset()
elif isinstance(error, MplexStreamEOF):
await stream.close()
finally:
if has_error:
await self.disconnect_peer(peer_id)
return
self.logger.debug("Received the hello message %s",
to_formatted_dict(hello_other_side))
try:
await self._validate_hello_req(hello_other_side)
except ValidationError as error:
self.logger.info(
"Handshake failed: hello message %s is invalid: %s",
to_formatted_dict(hello_other_side), str(error))
await stream.reset()
await self.say_goodbye(peer_id,
GoodbyeReasonCode.IRRELEVANT_NETWORK)
await self.disconnect_peer(peer_id)
return
hello_mine = self._make_hello_packet()
self.logger.debug("Sending our hello message %s",
to_formatted_dict(hello_mine))
try:
await write_resp(stream, hello_mine, ResponseCode.SUCCESS)
has_error = False
except (WriteMessageFailure, MplexStreamEOF,
MplexStreamReset) as error:
has_error = True
if isinstance(error, WriteMessageFailure):
await stream.reset()
elif isinstance(error, MplexStreamEOF):
await stream.close()
finally:
if has_error:
self.logger.info(
"Handshake failed: failed to write message %s",
hello_mine,
)
await self.disconnect_peer(peer_id)
return
if peer_id not in self.handshaked_peers:
peer = Peer.from_hello_request(self, peer_id, hello_other_side)
self.handshaked_peers.add(peer)
self.logger.debug(
"Handshake from %s is finished. Added to the `handshake_peers`",
peer_id,
)
# Check if we are behind the peer
self._compare_chain_tip_and_finalized_epoch(
hello_other_side.finalized_epoch,
hello_other_side.head_slot,
)
await stream.close()
