def generate_genesis_state(cls, get_genesis_time: Callable[[], Timestamp], network_dir: Path, keymap: Dict[Any, Any], clients: Tuple[Client, ...]) -> None: logger = cls.get_logger() state_machine_class = XiaoLongBaoStateMachine # Since create_mock_genesis takes a long time, update the real genesis_time later dummy_time = Timestamp(int(time.time())) state, _ = create_mock_genesis( num_validators=len(keymap.keys()), config=state_machine_class.config, keymap=keymap, genesis_block_class=state_machine_class.block_class, genesis_time=dummy_time, ) genesis_time = get_genesis_time() logger.info( "Genesis time will be %s from now", humanize_seconds(genesis_time - int(time.time())), ) state = state.copy(genesis_time=genesis_time, ) # The output here can be trusted, so use unsafe mode for performance yaml = YAML(typ='unsafe') with open(network_dir / GENESIS_FILE, "w") as f: yaml.dump(to_formatted_dict(state), f) # Distribute genesis file to clients for client in clients: with open(client.client_dir / GENESIS_FILE, "w") as f: yaml.dump(to_formatted_dict(state), f)
def generate_genesis_state(cls, get_genesis_time: Callable[[], Timestamp], network_dir: Path, keymap: Dict[BLSPubkey, int], clients: Tuple[Client, ...]) -> None: logger = cls.get_logger() state_machine_class = XiaoLongBaoStateMachine # NOTE: see https://github.com/ethereum/trinity/issues/786 override_lengths(XiaoLongBaoStateMachine.config) # Since create_mock_genesis takes a long time, update the real genesis_time later dummy_time = Timestamp(int(time.time())) state, _ = create_mock_genesis( pubkeys=cast( Sequence[BLSPubkey], keymap.keys(), ), config=state_machine_class.config, keymap=keymap, genesis_block_class=state_machine_class.block_class, genesis_time=dummy_time, ) genesis_time = get_genesis_time() logger.info( "Genesis time will be %s from now", humanize_seconds(genesis_time - int(time.time())), ) state = state.copy(genesis_time=genesis_time, ) # The output here can be trusted, so use unsafe mode for performance yaml = YAML(typ='unsafe') with open(network_dir / GENESIS_FILE, "w") as f: yaml.dump(to_formatted_dict(state), f) # Distribute genesis file to clients for client in clients: with open(client.client_dir / GENESIS_FILE, "w") as f: yaml.dump(to_formatted_dict(state), f)
def generate_genesis_state(cls, genesis_delay: Second, network_dir: Path, keymap: Dict[Any, Any], clients: Tuple[Client, ...]) -> None: logger = cls.get_logger() state_machine_class = XiaoLongBaoStateMachine # Since create_mock_genesis takes a long time, update the real genesis_time later dummy_time = Timestamp(int(time.time())) state, _ = create_mock_genesis( num_validators=len(keymap.keys()), config=state_machine_class.config, keymap=keymap, genesis_block_class=state_machine_class.block_class, genesis_time=dummy_time, ) logger.info(f"Genesis time will be {genesis_delay} seconds from now") genesis_time = Timestamp(int(time.time()) + genesis_delay) state = state.copy(genesis_time=genesis_time, ) yaml = YAML() with open(network_dir / GENESIS_FILE, "w") as f: yaml.dump(to_formatted_dict(state), f) # Distribute genesis file to clients for client in clients: with open(client.client_dir / GENESIS_FILE, "w") as f: yaml.dump(to_formatted_dict(state), f)
def validate_state(post_state: BeaconState, expected_state: BeaconState) -> None: # Use dict diff, easier to see the diff dict_post_state = to_formatted_dict(post_state, BeaconState) dict_expected_state = to_formatted_dict(expected_state, BeaconState) for key, value in dict_expected_state.items(): if isinstance(value, list): value = tuple(value) if dict_post_state[key] != value: raise AssertionError(f"state.{key} is incorrect:\n" f"\tExpected: {value}\n" f"\tResult: {dict_post_state[key]}\n")
def test_not_serializable(): octopi = (octopus, octopus, octopus) sedes = List(Animal, 2**32) output = to_formatted_dict(octopi, sedes) hashable_octopi = HashableList.from_iterable(octopi, sedes) assert hashable_octopi == from_formatted_dict(output, List(Animal, 2**32))
async def getCanonicalBlockBySlot(self, slot: Slot) -> Dict[Any, Any]: """ Return the canonical block of the given slot. """ block = await self.chain.coro_get_canonical_block_by_slot( slot, SignedBeaconBlock) return to_formatted_dict(block, sedes=SignedBeaconBlock)
async def getBlockByRoot(self, root: Root) -> Dict[Any, Any]: """ Return the block of given root. """ block = await self.chain.coro_get_block_by_root( root, SignedBeaconBlock) return to_formatted_dict(block, sedes=SignedBeaconBlock)
async def block(self, request: web.Request) -> Dict[str, Any]: if 'slot' in request.query: slot = Slot(int(request.query['slot'])) block = self.chain.get_canonical_block_by_slot(slot) elif 'root' in request.query: root = cast(Root, decode_hex(request.query['root'])) block = self.chain.get_block_by_root(root) return to_formatted_dict(block, sedes=BeaconBlock)
async def state(self, request: web.Request) -> Dict[str, Any]: if 'slot' in request.query: slot = Slot(int(request.query['slot'])) state = self.chain.get_state_by_slot(slot) elif 'root' in request.query: root = cast(Root, decode_hex(request.query['root'])) state = self.chain.get_state_by_root(root) else: raise APIServerError(f"Wrong querystring: {request.query}") return to_formatted_dict(state)
def test_custom_codec(): class CustomCodec(DefaultCodec): @staticmethod def format_integer(value, sedes): return encode_hex(sedes.serialize(value)) @staticmethod def unformat_integer(value, sedes): return sedes.deserialize(decode_hex(value)) output = to_formatted_dict(zoo, codec=CustomCodec) read_zoo = from_formatted_dict(output, Zoo, CustomCodec) assert read_zoo == zoo
def main(): parser = argparse.ArgumentParser() parser.add_argument("-ssz", type=str, required=True) parser.add_argument("-config", type=str, required=True) args = parser.parse_args() config_path = Path(args.config) minimal_config = load_config_at_path(config_path) override_lengths(minimal_config) with open(args.ssz, "rb") as f: encoded = f.read() state = ssz.decode(encoded, sedes=BeaconState) yaml = YAML(typ="unsafe") yaml.dump(to_formatted_dict(state), sys.stdout)
def execute_state_transtion(test_case, base_db): bls.use_noop_backend() dict_config = test_case['config'] verify_signatures = test_case['verify_signatures'] dict_initial_state = test_case['initial_state'] dict_blocks = test_case['blocks'] dict_expected_state = test_case['expected_state'] # TODO: make it case by case assert verify_signatures is False # Set config config = generate_config_by_dict(dict_config) # Set Vector fields override_vector_lengths(config) # Set pre_state pre_state = from_formatted_dict(dict_initial_state, BeaconState) # Set blocks blocks = () for dict_block in dict_blocks: block = from_formatted_dict(dict_block, SerenityBeaconBlock) blocks += (block, ) sm_class = SerenityStateMachine.configure( __name__='SerenityStateMachineForTesting', config=config, ) chaindb = BeaconChainDB(base_db, Eth2GenesisConfig(config)) attestation_pool = AttestationPool() post_state = pre_state.copy() for block in blocks: sm = sm_class(chaindb, attestation_pool, None, post_state) post_state, _ = sm.import_block(block) # Use dict diff, easier to see the diff dict_post_state = to_formatted_dict(post_state, BeaconState) for key, value in dict_expected_state.items(): if isinstance(value, list): value = tuple(value) assert dict_post_state[key] == value
async def _handle_goodbye(self, stream: INetStream) -> None: peer_id = stream.mplex_conn.peer_id self.logger.debug("Waiting for goodbye from %s", peer_id) try: goodbye = await read_req(stream, Goodbye) has_error = False except (ReadMessageFailure, MplexStreamEOF, MplexStreamReset) as error: has_error = True if isinstance(error, ReadMessageFailure): await stream.reset() elif isinstance(error, MplexStreamEOF): await stream.close() self.logger.debug("Received the goodbye message %s", to_formatted_dict(goodbye)) if not has_error: await stream.close() await self.disconnect_peer(peer_id)
def test_dump_serializble_with_explicit_sedes(): to_formatted_dict(zoo, Zoo)
async def say_hello(self, peer_id: ID) -> None: self.logger.info("Say hello to %s", str(peer_id)) hello_mine = self._make_hello_packet() self.logger.debug( "Opening new stream to peer=%s with protocols=%s", peer_id, [REQ_RESP_HELLO_SSZ], ) stream = await self.host.new_stream(peer_id, [REQ_RESP_HELLO_SSZ]) self.logger.debug("Sending our hello message %s", to_formatted_dict(hello_mine)) try: await write_req(stream, hello_mine) has_error = False except (WriteMessageFailure, MplexStreamEOF, MplexStreamReset) as error: has_error = True if isinstance(error, WriteMessageFailure): await stream.reset() elif isinstance(error, MplexStreamEOF): await stream.close() finally: if has_error: await self.disconnect_peer(peer_id) error_msg = f"fail to write request={hello_mine}" self.logger.info("Handshake failed: %s", error_msg) raise HandshakeFailure(error_msg) self.logger.debug("Waiting for hello from the other side") try: resp_code, hello_other_side = await read_resp(stream, HelloRequest) has_error = False except (ReadMessageFailure, MplexStreamEOF, MplexStreamReset) as error: has_error = True if isinstance(error, ReadMessageFailure): await stream.reset() elif isinstance(error, MplexStreamEOF): await stream.close() finally: if has_error: await self.disconnect_peer(peer_id) self.logger.info("Handshake failed: fail to read the response") raise HandshakeFailure("fail to read the response") self.logger.debug( "Received the hello message %s, resp_code=%s", to_formatted_dict(hello_other_side), resp_code, ) # TODO: Handle the case when `resp_code` is not success. if resp_code != ResponseCode.SUCCESS: # TODO: Do something according to the `ResponseCode` error_msg = ( "resp_code != ResponseCode.SUCCESS, " f"resp_code={resp_code}, error_msg={to_formatted_dict(hello_other_side)}" ) self.logger.info("Handshake failed: %s", error_msg) await stream.reset() await self.disconnect_peer(peer_id) raise HandshakeFailure(error_msg) hello_other_side = cast(HelloRequest, hello_other_side) try: await self._validate_hello_req(hello_other_side) except ValidationError as error: error_msg = ( f"hello message {to_formatted_dict(hello_other_side)} is invalid: {str(error)}" ) self.logger.info( "Handshake failed: %s. Disconnecting %s", error_msg, peer_id, ) await stream.reset() await self.say_goodbye(peer_id, GoodbyeReasonCode.IRRELEVANT_NETWORK) await self.disconnect_peer(peer_id) raise HandshakeFailure(error_msg) from error if peer_id not in self.handshaked_peers: peer = Peer.from_hello_request(self, peer_id, hello_other_side) self.handshaked_peers.add(peer) self.logger.debug( "Handshake to peer=%s is finished. Added to the `handshake_peers`", peer_id, ) # Check if we are behind the peer self._compare_chain_tip_and_finalized_epoch( hello_other_side.finalized_epoch, hello_other_side.head_slot, ) await stream.close()
async def getFinalizedHead(self) -> Dict[Any, Any]: """ Return finalized head block. """ block = await self.chain.coro_get_finalized_head(SignedBeaconBlock) return to_formatted_dict(block, sedes=SignedBeaconBlock)
def munge_all_args(cls, args: Namespace, trinity_config: TrinityConfig) -> None: logger = cls.get_logger() logger.info("Configuring testnet") override_lengths(MINIMAL_SERENITY_CONFIG) if args.wipedb: beacon_config = trinity_config.get_app_config(BeaconAppConfig) logger.info(f'Blowing away the database: {beacon_config.database_dir}') try: shutil.rmtree(beacon_config.database_dir) except FileNotFoundError: # there's nothing to wipe, that's fine! pass else: beacon_config.database_dir.mkdir() genesis_path = args.genesis_state_ssz_path or Path('resources/genesis.ssz') logger.info(f"Using genesis from {genesis_path}") # read the genesis! try: with open(genesis_path, 'rb') as f: # type: IO[Any] encoded = f.read() state = ssz.decode(encoded, sedes=BeaconState) except FileNotFoundError: import os logger.critical( "Required: the genesis state at %s/genesis.ssz," " or the path to this state with command line" " argument `--genesis-state-ssz-path $PATH`", os.getcwd(), ) sys.exit(1) now = int(time.time()) if args.start_time: if args.start_time <= now: logger.warning(f"--start-time must be a time in the future. Current time is {now}") delta = args.start_time - now logger.info("Time will begin %d seconds from now", delta) # adapt the state, then print the new root! state = state.copy( genesis_time=args.start_time ) elif args.start_delay: if args.start_delay < 0: logger.info(f"--start-time must be positive") sys.exit(1) start_time = now + args.start_delay logger.info("Genesis time is %d", start_time) state = state.copy( genesis_time=start_time ) else: logger.info("Using genesis_time from genesis state to determine start time") logger.info(f"Genesis hash tree root: {state.hash_tree_root.hex()}") logger.info(f"Configuring {trinity_config.trinity_root_dir}") # Save the genesis state to the data dir! yaml = YAML(typ='unsafe') with open(trinity_config.trinity_root_dir / GENESIS_FILE, 'w') as f: yaml.dump(to_formatted_dict(state), f) # Save the validator keys to the data dir keys_dir = trinity_config.trinity_root_dir / VALIDATOR_KEY_DIR try: shutil.rmtree(keys_dir) except FileNotFoundError: pass keys_dir.mkdir() def parse_key(maybe_hexstr: str) -> Union[int, str]: try: return to_int(hexstr=maybe_hexstr) except TypeError: return maybe_hexstr validators = args.validators if (args.validators or args.validators_from_yaml_key_file): if not validators: validators_keys_file = args.validators_from_yaml_key_file yaml = YAML(typ="unsafe") keys = yaml.load(Path(validators_keys_file)) for (i, key) in enumerate(keys): file_name = f"v_{i}.privkey" key_path = keys_dir / file_name with open(key_path, "w") as f: f.write(str(parse_key(key['privkey']))) else: validators = [int(token) for token in validators.split(',')] for validator in validators: if validator < 0 or validator > 15: logger.error(f"{validator} is not a valid validator") sys.exit(1) logger.info(f"Validating: {validators}") yaml = YAML(typ="unsafe") keys = yaml.load( Path( 'eth2/beacon/scripts/quickstart_state/keygen_16_validators.yaml' ) ) for validator_index in validators: key = keys[validator_index] file_name = f"v{validator_index:07d}.privkey" key_path = keys_dir / file_name with open(key_path, "w") as f: f.write(str(parse_key(key['privkey']))) else: logger.info("not running any validators") # disable some components which shouldn't be running args.disable_discovery = True args.disable_upnp = True args.network_tracking_backend = TrackingBackend.do_not_track
async def _handle_beacon_blocks(self, stream: INetStream) -> None: peer_id = stream.mplex_conn.peer_id if peer_id not in self.handshaked_peers: self.logger.info( "Processing beacon blocks request failed: not handshaked with peer=%s yet", peer_id, ) await stream.reset() return self.logger.debug( "Waiting for beacon blocks request from the other side") try: beacon_blocks_request = await read_req(stream, BeaconBlocksRequest) has_error = False except (ReadMessageFailure, MplexStreamEOF, MplexStreamReset) as error: has_error = True if isinstance(error, ReadMessageFailure): await stream.reset() elif isinstance(error, MplexStreamEOF): await stream.close() finally: if has_error: return self.logger.debug("Received the beacon blocks request message %s", to_formatted_dict(beacon_blocks_request)) try: requested_head_block = self.chain.get_block_by_hash_tree_root( beacon_blocks_request.head_block_root) except (BlockNotFound, ValidationError) as error: self.logger.info("Sending empty blocks, reason: %s", error) # We don't have the chain data peer is requesting requested_beacon_blocks: Tuple[BaseBeaconBlock, ...] = tuple() else: # Check if slot of specified head block is greater than specified start slot if requested_head_block.slot < beacon_blocks_request.start_slot: reason = ( f"Invalid request: head block slot({requested_head_block.slot})" f" lower than `start_slot`({beacon_blocks_request.start_slot})" ) try: await write_resp(stream, reason, ResponseCode.INVALID_REQUEST) has_error = False except (WriteMessageFailure, MplexStreamEOF, MplexStreamReset) as error: has_error = True if isinstance(error, WriteMessageFailure): await stream.reset() elif isinstance(error, MplexStreamEOF): await stream.close() finally: if has_error: self.logger.info( "Processing beacon blocks request failed: failed to write message %s", reason, ) return await stream.close() return else: try: requested_beacon_blocks = self._get_requested_beacon_blocks( beacon_blocks_request, requested_head_block) except ValidationError as val_error: reason = "Invalid request: " + str(val_error) try: await write_resp(stream, reason, ResponseCode.INVALID_REQUEST) has_error = False except (WriteMessageFailure, MplexStreamEOF, MplexStreamReset) as error: has_error = True if isinstance(error, WriteMessageFailure): await stream.reset() elif isinstance(error, MplexStreamEOF): await stream.close() finally: if has_error: self.logger.info( "Processing beacon blocks request failed: " "failed to write message %s", reason, ) return await stream.close() return # TODO: Should it be a successful response if peer is requesting # blocks on a fork we don't have data for? beacon_blocks_response = BeaconBlocksResponse( blocks=requested_beacon_blocks) self.logger.debug("Sending beacon blocks response %s", beacon_blocks_response) try: await write_resp(stream, beacon_blocks_response, ResponseCode.SUCCESS) has_error = False except (WriteMessageFailure, MplexStreamEOF, MplexStreamReset) as error: has_error = True if isinstance(error, WriteMessageFailure): await stream.reset() elif isinstance(error, MplexStreamEOF): await stream.close() finally: if has_error: self.logger.info( "Processing beacon blocks request failed: failed to write message %s", beacon_blocks_response, ) return self.logger.debug( "Processing beacon blocks request from %s is finished", peer_id, ) await stream.close()
async def head(self, request: web.Request) -> Dict[str, Any]: return to_formatted_dict(self.chain.get_canonical_head(), sedes=BeaconBlock)
def test_parsing_and_dumping(): json_str = json.dumps(to_formatted_dict(zoo)) read_zoo = from_formatted_dict(json.loads(json_str), Zoo) assert read_zoo == zoo
def test_not_serializable(): octopi = (octopus, octopus, octopus) output = to_formatted_dict(octopi, List(Animal)) assert octopi == from_formatted_dict(output, List(Animal))
async def _handle_hello(self, stream: INetStream) -> None: # TODO: Find out when we should respond the `ResponseCode` # other than `ResponseCode.SUCCESS`. peer_id = stream.mplex_conn.peer_id self.logger.debug("Waiting for hello from the other side") try: hello_other_side = await read_req(stream, HelloRequest) has_error = False except (ReadMessageFailure, MplexStreamEOF, MplexStreamReset) as error: has_error = True if isinstance(error, ReadMessageFailure): await stream.reset() elif isinstance(error, MplexStreamEOF): await stream.close() finally: if has_error: await self.disconnect_peer(peer_id) return self.logger.debug("Received the hello message %s", to_formatted_dict(hello_other_side)) try: await self._validate_hello_req(hello_other_side) except ValidationError as error: self.logger.info( "Handshake failed: hello message %s is invalid: %s", to_formatted_dict(hello_other_side), str(error)) await stream.reset() await self.say_goodbye(peer_id, GoodbyeReasonCode.IRRELEVANT_NETWORK) await self.disconnect_peer(peer_id) return hello_mine = self._make_hello_packet() self.logger.debug("Sending our hello message %s", to_formatted_dict(hello_mine)) try: await write_resp(stream, hello_mine, ResponseCode.SUCCESS) has_error = False except (WriteMessageFailure, MplexStreamEOF, MplexStreamReset) as error: has_error = True if isinstance(error, WriteMessageFailure): await stream.reset() elif isinstance(error, MplexStreamEOF): await stream.close() finally: if has_error: self.logger.info( "Handshake failed: failed to write message %s", hello_mine, ) await self.disconnect_peer(peer_id) return if peer_id not in self.handshaked_peers: peer = Peer.from_hello_request(self, peer_id, hello_other_side) self.handshaked_peers.add(peer) self.logger.debug( "Handshake from %s is finished. Added to the `handshake_peers`", peer_id, ) # Check if we are behind the peer self._compare_chain_tip_and_finalized_epoch( hello_other_side.finalized_epoch, hello_other_side.head_slot, ) await stream.close()