async def do_run(cls, boot_info: BootInfo, event_bus: EndpointAPI) -> None: trinity_config = boot_info.trinity_config beacon_app_config = trinity_config.get_app_config(BeaconAppConfig) chain_config = beacon_app_config.get_chain_config() base_db = DBClient.connect(trinity_config.database_ipc_path) # TODO: For now we use fake eth1 monitor. # if boot_info.args.eth1client_rpc: # w3: Web3 = Web3.HTTPProvider(boot_info.args.eth1client_rpc) # else: # w3: Web3 = None # TODO: For now we use fake eth1 monitor. So we load validators data from # interop setting and hardcode the deposit data into fake eth1 data provider. chain = chain_config.beacon_chain_class(base_db, chain_config.genesis_config) config = chain.get_state_machine().config key_set = load_yaml_at( Path( "eth2/beacon/scripts/quickstart_state/keygen_16_validators.yaml" )) pubkeys, privkeys, withdrawal_credentials = create_keypair_and_mock_withdraw_credentials( config, key_set # type: ignore ) initial_deposits = (create_mock_deposit_data( config=config, pubkey=pubkey, privkey=privkey, withdrawal_credentials=withdrawal_credential, ) for pubkey, privkey, withdrawal_credential in zip( pubkeys, privkeys, withdrawal_credentials)) # Set the timestamp of start block earlier enough so that eth1 monitor # can query up to 2 * `ETH1_FOLLOW_DISTANCE` of blocks in the beginning. start_block_timestamp = (chain_config.genesis_data.genesis_time - 3 * ETH1_FOLLOW_DISTANCE * AVERAGE_BLOCK_TIME) with base_db: fake_eth1_data_provider = FakeEth1DataProvider( start_block_number=START_BLOCK_NUMBER, start_block_timestamp=Timestamp(start_block_timestamp), num_deposits_per_block=NUM_DEPOSITS_PER_BLOCK, initial_deposits=tuple(initial_deposits), ) eth1_monitor_service: Service = Eth1Monitor( eth1_data_provider=fake_eth1_data_provider, num_blocks_confirmed=NUM_BLOCKS_CONFIRMED, polling_period=POLLING_PERIOD, start_block_number=BlockNumber(START_BLOCK_NUMBER - 1), event_bus=event_bus, base_db=base_db, ) try: await TrioManager.run_service(eth1_monitor_service) except Exception: await event_bus.broadcast( ShutdownRequest("Eth1 Monitor ended unexpectedly")) raise
async def do_run(self, event_bus: EndpointAPI) -> None: boot_info = self._boot_info config = boot_info.trinity_config db = DBClient.connect(config.database_ipc_path) if boot_info.args.disable_discovery: discovery_service: async_service.Service = NoopDiscoveryService( event_bus, ) else: vm_config = config.get_app_config( Eth1AppConfig).get_chain_config().vm_configuration headerdb = TrioHeaderDB(db) eth_cap_provider = functools.partial(generate_eth_cap_enr_field, vm_config, headerdb) socket = trio.socket.socket(family=trio.socket.AF_INET, type=trio.socket.SOCK_DGRAM) await socket.bind(("0.0.0.0", config.port)) base_db = LevelDB(config.enr_db_dir) enr_db = ENRDB(base_db) discovery_service = PreferredNodeDiscoveryService( config.nodekey, config.port, config.port, config.bootstrap_nodes, config.preferred_nodes, event_bus, socket, enr_db, (eth_cap_provider, ), ) with db: await async_service.run_trio_service(discovery_service)
async def do_run(cls, boot_info: BootInfo, event_bus: EndpointAPI) -> None: config = boot_info.trinity_config db = DBClient.connect(config.database_ipc_path) if boot_info.args.disable_discovery: discovery_service: async_service.Service = StaticDiscoveryService( event_bus, config.preferred_nodes, ) else: vm_config = config.get_app_config(Eth1AppConfig).get_chain_config().vm_configuration headerdb = TrioHeaderDB(db) eth_cap_provider = functools.partial(generate_eth_cap_enr_field, vm_config, headerdb) socket = trio.socket.socket(family=trio.socket.AF_INET, type=trio.socket.SOCK_DGRAM) await socket.bind(("0.0.0.0", config.port)) base_db = LevelDB(config.node_db_dir) node_db = NodeDB(default_identity_scheme_registry, base_db) discovery_service = PreferredNodeDiscoveryService( config.nodekey, config.port, config.port, config.bootstrap_nodes, config.preferred_nodes, event_bus, socket, node_db, (eth_cap_provider,), ) try: with db: await async_service.run_trio_service(discovery_service) except Exception: event_bus.broadcast_nowait(ShutdownRequest("Discovery ended unexpectedly")) raise
def do_start(self) -> None: trinity_config = self.boot_info.trinity_config db = DBClient.connect(trinity_config.database_ipc_path) app_config = trinity_config.get_app_config(Eth1AppConfig) chain_config = app_config.get_chain_config() chain = chain_config.full_chain_class(db) if self.boot_info.trinity_config.network_id == MAINNET_NETWORK_ID: validator = DefaultTransactionValidator(chain, BYZANTIUM_MAINNET_BLOCK) elif self.boot_info.trinity_config.network_id == ROPSTEN_NETWORK_ID: validator = DefaultTransactionValidator(chain, BYZANTIUM_ROPSTEN_BLOCK) else: # TODO: We could hint the user about e.g. a --tx-pool-no-validation flag to run the # tx pool without tx validation in this case raise ValueError( "The TxPool plugin only supports MainnetChain or RopstenChain") proxy_peer_pool = ETHProxyPeerPool(self.event_bus, TO_NETWORKING_BROADCAST_CONFIG) self.tx_pool = TxPool(self.event_bus, proxy_peer_pool, validator) asyncio.ensure_future( exit_with_services(self.tx_pool, self._event_bus_service)) asyncio.ensure_future(self.tx_pool.run())
async def fetch_witnesses( peer: ETHProxyPeer, block_hash: Hash32, block_number: BlockNumber, event_bus: EndpointAPI, database_ipc_path: pathlib.Path, metrics_registry: MetricsRegistry, logger: ExtendedDebugLogger, ) -> Tuple[Hash32, ...]: """ Fetch witness hashes for the given block from the given peer and emit a CollectMissingTrieNodes event to trigger the download of the trie nodes referred by them. """ block_str = f"<Block #{block_number}-0x{humanize_hash(block_hash)}>" try: logger.debug( "Attempting to fetch witness hashes for %s from %s", block_str, peer) witness_hashes = await peer.wit_api.get_block_witness_hashes(block_hash) except asyncio.TimeoutError: logger.debug( "Timed out trying to fetch witnesses for %s from %s", block_str, peer) return tuple() except Exception as err: logger.warning( "Error fetching witnesses for %s from %s: %s", block_str, peer, err) return tuple() else: if witness_hashes: logger.debug( "Got witness hashes for %s, asking BeamSyncer to fetch them", block_str) # XXX: Consider using urgent=False if the new block is more than a couple blocks ahead # of our tip, as otherwise when beam sync start to falls behind it may be more # difficult to catch up. urgent = True try: with trio.fail_after(1): # Sometimes we get a NewBlock/NewBlockHashes msg before the BeamSyncer service # has started, and there will be no subscribers to CollectMissingTrieNodes in # that case. This ensures we wait for it to start before attempting to fire # CollectMissingTrieNodes events. await event_bus.wait_until_any_endpoint_subscribed_to(CollectMissingTrieNodes) except trio.TooSlowError: logger.warning( "No subscribers for CollectMissingTrieNodes, cannot fetch witnesses for %s", block_str, ) return witness_hashes await event_bus.broadcast(CollectMissingTrieNodes(witness_hashes, urgent, block_number)) base_db = DBClient.connect(database_ipc_path) with base_db: wit_db = AsyncWitnessDB(base_db) wit_db.persist_witness_hashes(block_hash, witness_hashes) else: logger.debug( "%s announced %s but doesn't have witness hashes for it. " "This could be a peer that does not support the wit protocol, though", peer, block_str, ) return witness_hashes
async def do_run(cls, boot_info: BootInfo, event_bus: EndpointAPI) -> None: trinity_config = boot_info.trinity_config app_config = trinity_config.get_app_config(Eth1AppConfig) chain_config = app_config.get_chain_config() base_db = DBClient.connect(trinity_config.database_ipc_path) with base_db: loop = asyncio.get_event_loop() beam_chain = make_pausing_beam_chain( chain_config.vm_configuration, chain_config.chain_id, base_db, event_bus, # these preview executions are lower priority than the primary block import loop=loop, urgent=False, ) preview_server = BlockPreviewServer(event_bus, beam_chain, cls.shard_num) async with run_service(preview_server): await preview_server.cancellation()
async def do_run(cls, boot_info: BootInfo, event_bus: EndpointAPI) -> None: trinity_config = boot_info.trinity_config db = DBClient.connect(trinity_config.database_ipc_path) with db: app_config = trinity_config.get_app_config(Eth1AppConfig) chain_config = app_config.get_chain_config() chain = chain_config.full_chain_class(db) if boot_info.trinity_config.network_id == MAINNET_NETWORK_ID: validator = DefaultTransactionValidator( chain, ISTANBUL_MAINNET_BLOCK) elif boot_info.trinity_config.network_id == ROPSTEN_NETWORK_ID: validator = DefaultTransactionValidator( chain, ISTANBUL_ROPSTEN_BLOCK) elif boot_info.trinity_config.network_id == GOERLI_NETWORK_ID: validator = DefaultTransactionValidator( chain, ISTANBUL_GOERLI_BLOCK) else: raise Exception("This code path should not be reachable") proxy_peer_pool = ETHProxyPeerPool(event_bus, TO_NETWORKING_BROADCAST_CONFIG) async with background_asyncio_service(proxy_peer_pool): tx_pool = TxPool(event_bus, proxy_peer_pool, validator) async with background_asyncio_service(tx_pool) as manager: await manager.wait_finished()
def get_beacon_shell_context(database_dir: Path, trinity_config: TrinityConfig) -> Dict[str, Any]: app_config = trinity_config.get_app_config(BeaconAppConfig) ipc_path = trinity_config.database_ipc_path trinity_already_running = ipc_path.exists() if trinity_already_running: db = DBClient.connect(ipc_path) else: db = LevelDB(database_dir) chain_config = app_config.get_chain_config() chain = chain_config.beacon_chain_class attestation_pool = AttestationPool() chain = chain_config.beacon_chain_class(db, attestation_pool, chain_config.genesis_config) chaindb = BeaconChainDB(db, chain_config.genesis_config) head = chaindb.get_canonical_head(BeaconBlock) return { 'db': db, 'chaindb': chaindb, 'trinity_config': trinity_config, 'chain_config': chain_config, 'chain': chain, 'block_number': head.slot, 'hex_hash': head.hash_tree_root.hex(), 'state_root_hex': encode_hex(head.state_root), 'trinity_already_running': trinity_already_running }
async def do_run(cls, boot_info: BootInfo, event_bus: EndpointAPI) -> None: trinity_config = boot_info.trinity_config db = DBClient.connect(trinity_config.database_ipc_path) app_config = trinity_config.get_app_config(Eth1AppConfig) chain_config = app_config.get_chain_config() chain = chain_config.full_chain_class(db) if boot_info.trinity_config.network_id == MAINNET_NETWORK_ID: validator = DefaultTransactionValidator(chain, PETERSBURG_MAINNET_BLOCK) elif boot_info.trinity_config.network_id == ROPSTEN_NETWORK_ID: validator = DefaultTransactionValidator(chain, PETERSBURG_ROPSTEN_BLOCK) else: raise Exception("This code path should not be reachable") proxy_peer_pool = ETHProxyPeerPool(event_bus, TO_NETWORKING_BROADCAST_CONFIG) tx_pool = TxPool(event_bus, proxy_peer_pool, validator) async with run_service(tx_pool): await tx_pool.cancellation()
def do_start(self) -> None: trinity_config = self.boot_info.trinity_config db = DBClient.connect(trinity_config.database_ipc_path) app_config = trinity_config.get_app_config(Eth1AppConfig) chain_config = app_config.get_chain_config() chain = chain_config.full_chain_class(db) if self.boot_info.trinity_config.network_id == MAINNET_NETWORK_ID: validator = DefaultTransactionValidator(chain, PETERSBURG_MAINNET_BLOCK) elif self.boot_info.trinity_config.network_id == ROPSTEN_NETWORK_ID: validator = DefaultTransactionValidator(chain, PETERSBURG_ROPSTEN_BLOCK) else: raise ValueError( "The TxPool component only supports MainnetChain or RopstenChain" ) proxy_peer_pool = ETHProxyPeerPool(self.event_bus, TO_NETWORKING_BROADCAST_CONFIG) self.tx_pool = TxPool(self.event_bus, proxy_peer_pool, validator) asyncio.ensure_future( exit_with_services(self.tx_pool, self._event_bus_service)) asyncio.ensure_future(self.tx_pool.run())
def chain_for_beacon_config( trinity_config: TrinityConfig, beacon_app_config: BeaconAppConfig, ) -> Iterator[AsyncBeaconChainDB]: db = DBClient.connect(trinity_config.database_ipc_path) with db: yield AsyncBeaconChainDB(db)
def get_eth1_shell_context(database_dir: Path, trinity_config: TrinityConfig) -> Dict[str, Any]: app_config = trinity_config.get_app_config(Eth1AppConfig) ipc_path = trinity_config.database_ipc_path trinity_already_running = ipc_path.exists() if trinity_already_running: db = DBClient.connect(ipc_path) else: db = LevelDB(database_dir) chaindb = ChainDB(db) head = chaindb.get_canonical_head() chain_config = app_config.get_chain_config() chain = chain_config.full_chain_class(db) return { 'db': db, 'chaindb': chaindb, 'trinity_config': trinity_config, 'chain_config': chain_config, 'chain': chain, 'block_number': head.block_number, 'hex_hash': head.hex_hash, 'state_root_hex': encode_hex(head.state_root), 'trinity_already_running': trinity_already_running, }
async def do_run(self, event_bus: EndpointAPI) -> None: trinity_config = self._boot_info.trinity_config app_config = trinity_config.get_app_config(Eth1AppConfig) chain_config = app_config.get_chain_config() base_db = DBClient.connect(trinity_config.database_ipc_path) with base_db: loop = asyncio.get_event_loop() beam_chain = make_pausing_beam_chain( chain_config.vm_configuration, chain_config.chain_id, chain_config.consensus_context_class, base_db, event_bus, # these preview executions are lower priority than the primary block import loop=loop, urgent=False, ) preview_server = BlockPreviewServer(event_bus, beam_chain, self.shard_num) async with background_asyncio_service(preview_server) as manager: await manager.wait_finished()
async def do_run(self, event_bus: EndpointAPI) -> None: trinity_config = self._boot_info.trinity_config app_config = trinity_config.get_app_config(Eth1AppConfig) chain_config = app_config.get_chain_config() base_db = DBClient.connect(trinity_config.database_ipc_path) if self._boot_info.args.enable_metrics: metrics_service = metrics_service_from_args( self._boot_info.args, AsyncioMetricsService) else: # Use a NoopMetricsService so that no code branches need to be taken if metrics # are disabled metrics_service = NOOP_METRICS_SERVICE with base_db: beam_chain = make_pausing_beam_chain( chain_config.vm_configuration, chain_config.chain_id, chain_config.consensus_context_class, base_db, event_bus, metrics_service.registry, loop=asyncio.get_event_loop(), ) import_server = BlockImportServer(event_bus, beam_chain) async with background_asyncio_service(metrics_service): async with background_asyncio_service( import_server) as manager: await manager.wait_finished()
def _get_base_db(database_dir: Path, ipc_path: Path) -> Iterator[AtomicDatabaseAPI]: trinity_already_running = ipc_path.exists() if trinity_already_running: db = DBClient.connect(ipc_path) with db: yield db else: yield LevelDB(database_dir)
def get_discv5_topic(trinity_config: TrinityConfig, protocol: Type[Protocol]) -> bytes: db = DBClient.connect(trinity_config.database_ipc_path) header_db = HeaderDB(db) genesis_hash = header_db.get_canonical_block_hash( BlockNumber(GENESIS_BLOCK_NUMBER)) return get_v5_topic(protocol, genesis_hash)
async def test_json_rpc_http_server(aiohttp_raw_server, aiohttp_client, event_bus, base_db, ipc_path): manager = DBManager(base_db) with manager.run(ipc_path): # Set chaindb override_lengths(SERENITY_CONFIG) db = DBClient.connect(ipc_path) genesis_config = SERENITY_CONFIG chaindb = AsyncBeaconChainDB(db, genesis_config) fork_choice_scoring = HigherSlotScoring() genesis_state, genesis_block = create_mock_genesis( pubkeys=(), config=SERENITY_CONFIG, keymap=dict(), genesis_block_class=BeaconBlock, genesis_time=0, ) chaindb.persist_state(genesis_state) chaindb.persist_block( SignedBeaconBlock.create(message=genesis_block), SignedBeaconBlock, fork_choice_scoring, ) try: rpc = RPCServer(initialize_beacon_modules(chaindb, event_bus), chaindb, event_bus) raw_server = await aiohttp_raw_server( RPCHandler.handle(rpc.execute)) client = await aiohttp_client(raw_server) request_id = 1 request_data = { "jsonrpc": "2.0", "method": "beacon_head", "params": [], "id": request_id, } response = await client.post("/", json=request_data) response_data = await response.json() assert response_data["id"] == request_id result = response_data["result"] assert result["slot"] == 0 assert decode_hex( result["block_root"]) == genesis_block.hash_tree_root assert decode_hex( result["state_root"]) == genesis_state.hash_tree_root except KeyboardInterrupt: pass finally: await raw_server.close() db.close()
def __init__(self, event_bus: EndpointAPI, trinity_config: TrinityConfig) -> None: super().__init__() self.trinity_config = trinity_config self._base_db = DBClient.connect(trinity_config.database_ipc_path) self._headerdb = AsyncHeaderDB(self._base_db) self._jsonrpc_ipc_path: Path = trinity_config.jsonrpc_ipc_path self._network_id = trinity_config.network_id self.event_bus = event_bus
def __init__(self, event_bus: EndpointAPI, trinity_config: TrinityConfig) -> None: self.trinity_config = trinity_config self._base_db = DBClient.connect(trinity_config.database_ipc_path) self._headerdb = AsyncHeaderDB(self._base_db) self._jsonrpc_ipc_path: Path = trinity_config.jsonrpc_ipc_path self._network_id = trinity_config.network_id self.event_bus = event_bus self.master_cancel_token = CancelToken(type(self).__name__)
def test_db_manager_lifecycle_with_connections(db, ipc_path): manager = DBManager(db) assert not manager.is_running assert not manager.is_stopped with manager.run(ipc_path): assert manager.is_running assert not manager.is_stopped client_a = DBClient.connect(ipc_path) client_b = DBClient.connect(ipc_path) assert manager.is_running assert not manager.is_stopped client_a.close() client_b.close() assert not manager.is_running assert manager.is_stopped
async def do_run(cls, boot_info: BootInfo, event_bus: EndpointAPI) -> None: trinity_config = boot_info.trinity_config base_db = DBClient.connect(trinity_config.database_ipc_path) with base_db: if trinity_config.has_app_config(Eth1AppConfig): server = cls.make_eth1_request_server( trinity_config.get_app_config(Eth1AppConfig), base_db, event_bus, ) else: raise Exception("Trinity config must have eth1 config") await run_asyncio_service(server)
def do_start(self) -> None: trinity_config = self.boot_info.trinity_config base_db = DBClient.connect(trinity_config.database_ipc_path) if trinity_config.has_app_config(Eth1AppConfig): server = self.make_eth1_request_server( trinity_config.get_app_config(Eth1AppConfig), base_db, ) else: raise Exception("Trinity config must have eth1 config") asyncio.ensure_future(exit_with_services(server, self._event_bus_service)) asyncio.ensure_future(server.run())
async def do_run(self, event_bus: EndpointAPI) -> None: boot_info = self._boot_info trinity_config = boot_info.trinity_config base_db = DBClient.connect(trinity_config.database_ipc_path) with base_db: if trinity_config.has_app_config(Eth1AppConfig): server = self.make_eth1_request_server( trinity_config.get_app_config(Eth1AppConfig), base_db, event_bus, ) else: raise Exception("Trinity config must have eth1 config") async with background_asyncio_service(server) as manager: await manager.wait_finished()
def do_start(self) -> None: trinity_config = self.boot_info.trinity_config app_config = trinity_config.get_app_config(Eth1AppConfig) chain_config = app_config.get_chain_config() self._beam_chain = make_pausing_beam_chain( chain_config.vm_configuration, chain_config.chain_id, DBClient.connect(trinity_config.database_ipc_path), self.event_bus, self._loop, ) import_server = BlockImportServer(self.event_bus, self._beam_chain) asyncio.ensure_future(exit_with_services(import_server, self._event_bus_service)) asyncio.ensure_future(import_server.run())
def chain_for_eth1_config(self, trinity_config: TrinityConfig, eth1_app_config: Eth1AppConfig) -> AsyncChainAPI: chain_config = eth1_app_config.get_chain_config() db = DBClient.connect(trinity_config.database_ipc_path) if eth1_app_config.database_mode is Eth1DbMode.LIGHT: header_db = HeaderDB(db) event_bus_light_peer_chain = EventBusLightPeerChain(self.event_bus) return chain_config.light_chain_class( header_db, peer_chain=event_bus_light_peer_chain) elif eth1_app_config.database_mode is Eth1DbMode.FULL: return chain_config.full_chain_class(db) else: raise Exception( f"Unsupported Database Mode: {eth1_app_config.database_mode}")
def get_eth1_chain_with_remote_db( boot_info: BootInfo, event_bus: EndpointAPI) -> Iterator[ChainAPI]: app_config = boot_info.trinity_config.get_app_config(Eth1AppConfig) chain_config = app_config.get_chain_config() chain: ChainAPI base_db = DBClient.connect(boot_info.trinity_config.database_ipc_path) with base_db: if boot_info.args.sync_mode == SYNC_LIGHT: header_db = AsyncHeaderDB(base_db) chain = chain_config.light_chain_class( header_db, peer_chain=EventBusLightPeerChain(event_bus)) else: chain = chain_config.full_chain_class(base_db) yield chain
def get_chain(self) -> ChainAPI: app_config = self.boot_info.trinity_config.get_app_config( Eth1AppConfig) chain_config = app_config.get_chain_config() chain: ChainAPI base_db = DBClient.connect( self.boot_info.trinity_config.database_ipc_path) if self.boot_info.args.sync_mode == SYNC_LIGHT: header_db = AsyncHeaderDB(base_db) chain = chain_config.light_chain_class( header_db, peer_chain=EventBusLightPeerChain(self.event_bus)) else: chain = chain_config.full_chain_class(base_db) return chain
async def _fetch_witnesses( self, peer: ETHProxyPeer, block_hash: Hash32, block_number: BlockNumber) -> None: base_db = DBClient.connect(self._boot_info.trinity_config.database_ipc_path) with base_db: try: AsyncWitnessDB(base_db).get_witness_hashes(block_hash) except WitnessHashesUnavailable: pass else: block_str = f"Block #{block_number}-0x{humanize_hash(block_hash)}" self.logger.debug( "Already have witness hashes for %s, not fetching again", block_str) return await fetch_witnesses( peer, block_hash, block_number, self._event_bus, self._boot_info.trinity_config.database_ipc_path, self._metrics_registry, self.logger)
def chain_for_eth1_config(trinity_config: TrinityConfig, eth1_app_config: Eth1AppConfig, event_bus: EndpointAPI) -> Iterator[AsyncChainAPI]: chain_config = eth1_app_config.get_chain_config() db = DBClient.connect(trinity_config.database_ipc_path) with db: if eth1_app_config.database_mode is Eth1DbMode.LIGHT: header_db = HeaderDB(db) event_bus_light_peer_chain = EventBusLightPeerChain(event_bus) yield chain_config.light_chain_class( header_db, peer_chain=event_bus_light_peer_chain ) elif eth1_app_config.database_mode is Eth1DbMode.FULL: yield chain_config.full_chain_class(db) else: raise Exception(f"Unsupported Database Mode: {eth1_app_config.database_mode}")
def do_start(self) -> None: trinity_config = self.boot_info.trinity_config app_config = trinity_config.get_app_config(Eth1AppConfig) chain_config = app_config.get_chain_config() base_db = DBClient.connect(trinity_config.database_ipc_path) self._beam_chain = make_pausing_beam_chain( chain_config.vm_configuration, chain_config.chain_id, base_db, self.event_bus, self._loop, # these preview executions are lower priority than the primary block import urgent=False, ) import_server = BlockPreviewServer(self.event_bus, self._beam_chain, self.shard_num) asyncio.ensure_future(exit_with_services(import_server, self._event_bus_service)) asyncio.ensure_future(import_server.run())