async def test_light_syncer(request, event_loop, event_bus, chaindb_fresh, chaindb_20): client_peer, server_peer = await get_directly_linked_peers( request, event_loop, alice_peer_class=LESPeer, alice_headerdb=FakeAsyncHeaderDB(chaindb_fresh.db), bob_headerdb=FakeAsyncHeaderDB(chaindb_20.db)) client = LightChainSyncer(LatestTestChain(chaindb_fresh.db), chaindb_fresh, MockPeerPoolWithConnectedPeers([client_peer])) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=LESPeerPoolEventServer), run_request_server( event_bus, FakeAsyncChainDB(chaindb_20.db), server_type=LightRequestServer): server_peer.logger.info("%s is serving 20 blocks", server_peer) client_peer.logger.info("%s is syncing up 20", client_peer) def finalizer(): event_loop.run_until_complete(client.cancel()) # Yield control so that client/server.run() returns, otherwise asyncio will complain. event_loop.run_until_complete(asyncio.sleep(0.1)) request.addfinalizer(finalizer) asyncio.ensure_future(client.run()) await wait_for_head(chaindb_fresh, chaindb_20.get_canonical_head())
async def test_light_syncer(request, event_loop, event_bus, chaindb_fresh, chaindb_20): client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db) server_context = ChainContextFactory(headerdb__db=chaindb_20.db) peer_pair = LESV2PeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_peer, server_peer): client = LightChainSyncer( LatestTestChain(chaindb_fresh.db), chaindb_fresh, MockPeerPoolWithConnectedPeers([client_peer], event_bus=event_bus)) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=LESPeerPoolEventServer ), background_asyncio_service( LightRequestServer( event_bus, TO_NETWORKING_BROADCAST_CONFIG, AsyncChainDB(chaindb_20.db), )): server_peer.logger.info("%s is serving 20 blocks", server_peer) client_peer.logger.info("%s is syncing up 20", client_peer) async with background_asyncio_service(client): await wait_for_head(chaindb_fresh, chaindb_20.get_canonical_head())
def _make_syncer(self) -> LightChainSyncer: return LightChainSyncer( self.chain, self.headerdb, self.peer_pool, self.cancel_token, )
def get_p2p_server(self) -> LightChainSyncer: if self._p2p_server is None: if self.chain_class is None: raise AttributeError("LightNode subclass must set chain_class") self._p2p_server = LightChainSyncer( self.db_manager.get_chain(), # type: ignore self._headerdb, self._peer_pool, self.cancel_token) return self._p2p_server
async def test_light_syncer(request, event_loop, event_bus, chaindb_fresh, chaindb_20): client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db) server_context = ChainContextFactory(headerdb__db=chaindb_20.db) peer_pair = LESV2PeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_peer, server_peer): client = LightChainSyncer( LatestTestChain(chaindb_fresh.db), chaindb_fresh, MockPeerPoolWithConnectedPeers([client_peer], event_bus=event_bus)) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=LESPeerPoolEventServer ), background_asyncio_service( LightRequestServer( event_bus, TO_NETWORKING_BROADCAST_CONFIG, AsyncChainDB(chaindb_20.db), )): server_peer.logger.info("%s is serving 20 blocks", server_peer) client_peer.logger.info("%s is syncing up 20", client_peer) def finalizer(): event_loop.run_until_complete(client.cancel()) # Yield control so that client/server.run() returns, otherwise # asyncio will complain. event_loop.run_until_complete(asyncio.sleep(0.1)) request.addfinalizer(finalizer) asyncio.ensure_future(client.run()) await wait_for_head(chaindb_fresh, chaindb_20.get_canonical_head())
async def sync(self, logger: Logger, chain: BaseChain, db_manager: BaseManager, peer_pool: BaseChainPeerPool, cancel_token: CancelToken) -> None: syncer = LightChainSyncer( chain, db_manager.get_headerdb(), # type: ignore cast(LESPeerPool, peer_pool), cancel_token, ) await syncer.run()
async def sync(self, args: Namespace, logger: logging.Logger, chain: AsyncChainAPI, base_db: AtomicDatabaseAPI, peer_pool: BasePeerPool, event_bus: EndpointAPI) -> None: syncer = LightChainSyncer( chain, AsyncHeaderDB(base_db), cast(LESPeerPool, peer_pool), ) async with background_asyncio_service(syncer) as manager: await manager.wait_finished()
async def sync(self, args: Namespace, logger: Logger, chain: AsyncChainAPI, base_db: AtomicDatabaseAPI, peer_pool: BasePeerPool, event_bus: EndpointAPI, cancel_token: CancelToken) -> None: syncer = LightChainSyncer( chain, AsyncHeaderDB(base_db), cast(LESPeerPool, peer_pool), cancel_token, ) await syncer.run()
async def test_light_syncer(request, event_loop, chaindb_fresh, chaindb_20): client_peer, server_peer = await get_directly_linked_peers( request, event_loop, LESPeer, FakeAsyncHeaderDB(chaindb_fresh.db), LESPeer, FakeAsyncHeaderDB(chaindb_20.db)) client = LightChainSyncer(FrontierTestChain(chaindb_fresh.db), chaindb_fresh, MockPeerPoolWithConnectedPeers([client_peer])) server = LightChainSyncer(FrontierTestChain(chaindb_20.db), chaindb_20, MockPeerPoolWithConnectedPeers([server_peer])) asyncio.ensure_future(server.run()) def finalizer(): event_loop.run_until_complete( asyncio.gather( client.cancel(), server.cancel(), loop=event_loop, )) # Yield control so that client/server.run() returns, otherwise asyncio will complain. event_loop.run_until_complete(asyncio.sleep(0.1)) request.addfinalizer(finalizer) asyncio.ensure_future(client.run()) await wait_for_head(client.db, server.db.get_canonical_head())
async def test_lightchain_integration(request, event_loop, caplog, geth_ipc_path, enode, geth_process): """Test LightChainSyncer/LightPeerChain against a running geth instance. In order to run this manually, you can use `tox -e py36-lightchain_integration` or: pytest --integration --capture=no tests/integration/test_lightchain_integration.py The fixture for this test was generated with: geth --testnet --syncmode full It only needs the first 11 blocks for this test to succeed. """ if not pytest.config.getoption("--integration"): pytest.skip("Not asked to run integration tests") # will almost certainly want verbose logging in a failure caplog.set_level(logging.DEBUG) # make sure geth has been launched wait_for_socket(geth_ipc_path) remote = Node.from_uri(enode) base_db = AtomicDB() chaindb = FakeAsyncChainDB(base_db) chaindb.persist_header(ROPSTEN_GENESIS_HEADER) headerdb = FakeAsyncHeaderDB(base_db) context = ChainContext( headerdb=headerdb, network_id=ROPSTEN_NETWORK_ID, vm_configuration=ROPSTEN_VM_CONFIGURATION, ) peer_pool = LESPeerPool( privkey=ecies.generate_privkey(), context=context, ) chain = FakeAsyncRopstenChain(base_db) syncer = LightChainSyncer(chain, chaindb, peer_pool) syncer.min_peers_to_sync = 1 peer_chain = LightPeerChain(headerdb, peer_pool) server_request_handler = LightRequestServer(headerdb, peer_pool) asyncio.ensure_future(peer_pool.run()) asyncio.ensure_future(connect_to_peers_loop(peer_pool, tuple([remote]))) asyncio.ensure_future(peer_chain.run()) asyncio.ensure_future(server_request_handler.run()) asyncio.ensure_future(syncer.run()) await asyncio.sleep( 0) # Yield control to give the LightChainSyncer a chance to start def finalizer(): event_loop.run_until_complete(peer_pool.cancel()) event_loop.run_until_complete(peer_chain.cancel()) event_loop.run_until_complete(syncer.cancel()) event_loop.run_until_complete(server_request_handler.cancel()) request.addfinalizer(finalizer) n = 11 # Wait for the chain to sync a few headers. async def wait_for_header_sync(block_number): while headerdb.get_canonical_head().block_number < block_number: await asyncio.sleep(0.1) await asyncio.wait_for(wait_for_header_sync(n), 5) # https://ropsten.etherscan.io/block/11 header = headerdb.get_canonical_block_header_by_number(n) body = await peer_chain.coro_get_block_body_by_hash(header.hash) assert len(body['transactions']) == 15 receipts = await peer_chain.coro_get_receipts(header.hash) assert len(receipts) == 15 assert encode_hex(keccak(rlp.encode(receipts[0]))) == ( '0xf709ed2c57efc18a1675e8c740f3294c9e2cb36ba7bb3b89d3ab4c8fef9d8860') assert len(peer_pool) == 1 peer = peer_pool.highest_td_peer head = await peer_chain.coro_get_block_header_by_hash(peer.head_hash) # In order to answer queries for contract code, geth needs the state trie entry for the block # we specify in the query, but because of fast sync we can only assume it has that for recent # blocks, so we use the current head to lookup the code for the contract below. # https://ropsten.etherscan.io/address/0x95a48dca999c89e4e284930d9b9af973a7481287 contract_addr = decode_hex('0x8B09D9ac6A4F7778fCb22852e879C7F3B2bEeF81') contract_code = await peer_chain.coro_get_contract_code( head.hash, contract_addr) assert encode_hex(contract_code) == '0x600060006000600060006000356000f1' account = await peer_chain.coro_get_account(head.hash, contract_addr) assert account.code_hash == keccak(contract_code) assert account.balance == 0
async def _main() -> None: parser = argparse.ArgumentParser() parser.add_argument('-db', type=str, required=True) parser.add_argument('-light', action="store_true") parser.add_argument('-nodekey', type=str) parser.add_argument('-enode', type=str, required=False, help="The enode we should connect to") parser.add_argument('-debug', action="store_true") args = parser.parse_args() logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s: %(message)s', datefmt='%H:%M:%S') log_level = logging.INFO if args.debug: log_level = logging.DEBUG loop = asyncio.get_event_loop() base_db = LevelDB(args.db) headerdb = AsyncHeaderDB(AtomicDB(base_db)) chaindb = AsyncChainDB(AtomicDB(base_db)) try: genesis = chaindb.get_canonical_block_header_by_number(BlockNumber(0)) except HeaderNotFound: genesis = ROPSTEN_GENESIS_HEADER chaindb.persist_header(genesis) peer_pool_class: Type[Union[ETHPeerPool, LESPeerPool]] = ETHPeerPool if args.light: peer_pool_class = LESPeerPool chain_class: Union[Type[AsyncRopstenChain], Type[AsyncMainnetChain]] if genesis.hash == ROPSTEN_GENESIS_HEADER.hash: chain_id = RopstenChain.chain_id vm_config = ROPSTEN_VM_CONFIGURATION chain_class = AsyncRopstenChain elif genesis.hash == MAINNET_GENESIS_HEADER.hash: chain_id = MainnetChain.chain_id vm_config = MAINNET_VM_CONFIGURATION # type: ignore chain_class = AsyncMainnetChain else: raise RuntimeError("Unknown genesis: %s", genesis) if args.nodekey: privkey = load_nodekey(Path(args.nodekey)) else: privkey = ecies.generate_privkey() context = ChainContext( headerdb=headerdb, network_id=chain_id, vm_configuration=vm_config, client_version_string=construct_trinity_client_identifier(), listen_port=30309, p2p_version=DEVP2P_V5, ) peer_pool = peer_pool_class(privkey=privkey, context=context) if args.enode: nodes = tuple([Node.from_uri(args.enode)]) else: nodes = DEFAULT_PREFERRED_NODES[chain_id] async with background_asyncio_service(peer_pool) as manager: manager.run_task(connect_to_peers_loop(peer_pool, nodes)) # type: ignore chain = chain_class(base_db) syncer: Service = None if args.light: syncer = LightChainSyncer(chain, headerdb, cast(LESPeerPool, peer_pool)) else: syncer = RegularChainSyncer(chain, chaindb, cast(ETHPeerPool, peer_pool)) logging.getLogger().setLevel(log_level) sigint_received = asyncio.Event() for sig in [signal.SIGINT, signal.SIGTERM]: loop.add_signal_handler(sig, sigint_received.set) async def exit_on_sigint() -> None: await sigint_received.wait() syncer.get_manager().cancel() asyncio.ensure_future(exit_on_sigint()) async with background_asyncio_service(syncer) as syncer_manager: await syncer_manager.wait_finished()
def _test() -> None: import argparse from pathlib import Path import signal from p2p import ecies from p2p.kademlia import Node from eth.chains.ropsten import RopstenChain, ROPSTEN_GENESIS_HEADER, ROPSTEN_VM_CONFIGURATION from eth.chains.mainnet import MainnetChain, MAINNET_GENESIS_HEADER, MAINNET_VM_CONFIGURATION from eth.db.backends.level import LevelDB from tests.trinity.core.integration_test_helpers import ( FakeAsyncChainDB, FakeAsyncMainnetChain, FakeAsyncRopstenChain, FakeAsyncHeaderDB, connect_to_peers_loop) from trinity.constants import DEFAULT_PREFERRED_NODES from trinity.protocol.common.context import ChainContext from trinity._utils.chains import load_nodekey parser = argparse.ArgumentParser() parser.add_argument('-db', type=str, required=True) parser.add_argument('-fast', action="store_true") parser.add_argument('-light', action="store_true") parser.add_argument('-nodekey', type=str) parser.add_argument('-enode', type=str, required=False, help="The enode we should connect to") parser.add_argument('-debug', action="store_true") args = parser.parse_args() logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s: %(message)s', datefmt='%H:%M:%S') log_level = logging.INFO if args.debug: log_level = logging.DEBUG loop = asyncio.get_event_loop() base_db = LevelDB(args.db) headerdb = FakeAsyncHeaderDB(base_db) chaindb = FakeAsyncChainDB(base_db) try: genesis = chaindb.get_canonical_block_header_by_number(0) except HeaderNotFound: genesis = ROPSTEN_GENESIS_HEADER chaindb.persist_header(genesis) peer_pool_class: Type[Union[ETHPeerPool, LESPeerPool]] = ETHPeerPool if args.light: peer_pool_class = LESPeerPool if genesis.hash == ROPSTEN_GENESIS_HEADER.hash: network_id = RopstenChain.network_id vm_config = ROPSTEN_VM_CONFIGURATION # type: ignore chain_class = FakeAsyncRopstenChain elif genesis.hash == MAINNET_GENESIS_HEADER.hash: network_id = MainnetChain.network_id vm_config = MAINNET_VM_CONFIGURATION # type: ignore chain_class = FakeAsyncMainnetChain else: raise RuntimeError("Unknown genesis: %s", genesis) if args.nodekey: privkey = load_nodekey(Path(args.nodekey)) else: privkey = ecies.generate_privkey() context = ChainContext( headerdb=headerdb, network_id=network_id, vm_configuration=vm_config, ) peer_pool = peer_pool_class(privkey=privkey, context=context) if args.enode: nodes = tuple([Node.from_uri(args.enode)]) else: nodes = DEFAULT_PREFERRED_NODES[network_id] asyncio.ensure_future(peer_pool.run()) peer_pool.run_task(connect_to_peers_loop(peer_pool, nodes)) chain = chain_class(base_db) syncer: BaseHeaderChainSyncer = None if args.fast: syncer = FastChainSyncer(chain, chaindb, cast(ETHPeerPool, peer_pool)) elif args.light: syncer = LightChainSyncer(chain, headerdb, cast(LESPeerPool, peer_pool)) else: syncer = RegularChainSyncer(chain, chaindb, cast(ETHPeerPool, peer_pool)) syncer.logger.setLevel(log_level) syncer.min_peers_to_sync = 1 sigint_received = asyncio.Event() for sig in [signal.SIGINT, signal.SIGTERM]: loop.add_signal_handler(sig, sigint_received.set) async def exit_on_sigint() -> None: await sigint_received.wait() await peer_pool.cancel() await syncer.cancel() loop.stop() async def run() -> None: await syncer.run() syncer.logger.info("run() finished, exiting") sigint_received.set() # loop.set_debug(True) asyncio.ensure_future(exit_on_sigint()) asyncio.ensure_future(run()) loop.run_forever() loop.close()
def _make_syncer(self, peer_pool: PeerPool) -> BaseService: return LightChainSyncer(self.chain, self.headerdb, peer_pool, self.cancel_token)
async def test_lightchain_integration(request, event_loop, caplog): """Test LightChainSyncer/LightPeerChain against a running geth instance. In order to run this you need to pass the following to pytest: pytest --integration --capture=no --enode=... If you don't have any geth testnet data ready, it is very quick to generate some with: geth --testnet --syncmode full You only need the first 11 blocks for this test to succeed. Then you can restart geth with: geth --testnet --lightserv 90 --nodiscover """ # TODO: Implement a pytest fixture that runs geth as above, so that we don't need to run it # manually. if not pytest.config.getoption("--integration"): pytest.skip("Not asked to run integration tests") # will almost certainly want verbose logging in a failure caplog.set_level(logging.DEBUG) remote = Node.from_uri(pytest.config.getoption("--enode")) base_db = MemoryDB() chaindb = FakeAsyncChainDB(base_db) chaindb.persist_header(ROPSTEN_GENESIS_HEADER) headerdb = FakeAsyncHeaderDB(base_db) peer_pool = PeerPool( LESPeer, FakeAsyncHeaderDB(base_db), ROPSTEN_NETWORK_ID, ecies.generate_privkey(), ROPSTEN_VM_CONFIGURATION, ) chain = FakeAsyncRopstenChain(base_db) syncer = LightChainSyncer(chain, chaindb, peer_pool) syncer.min_peers_to_sync = 1 peer_chain = LightPeerChain(headerdb, peer_pool) asyncio.ensure_future(peer_pool.run()) asyncio.ensure_future(connect_to_peers_loop(peer_pool, tuple([remote]))) asyncio.ensure_future(peer_chain.run()) asyncio.ensure_future(syncer.run()) await asyncio.sleep( 0) # Yield control to give the LightChainSyncer a chance to start def finalizer(): event_loop.run_until_complete(peer_pool.cancel()) event_loop.run_until_complete(peer_chain.cancel()) event_loop.run_until_complete(syncer.cancel()) request.addfinalizer(finalizer) n = 11 # Wait for the chain to sync a few headers. async def wait_for_header_sync(block_number): while headerdb.get_canonical_head().block_number < block_number: await asyncio.sleep(0.1) await asyncio.wait_for(wait_for_header_sync(n), 5) # https://ropsten.etherscan.io/block/11 header = headerdb.get_canonical_block_header_by_number(n) body = await peer_chain.get_block_body_by_hash(header.hash) assert len(body['transactions']) == 15 receipts = await peer_chain.get_receipts(header.hash) assert len(receipts) == 15 assert encode_hex(keccak(rlp.encode(receipts[0]))) == ( '0xf709ed2c57efc18a1675e8c740f3294c9e2cb36ba7bb3b89d3ab4c8fef9d8860') assert len(peer_pool) == 1 head_info = peer_pool.highest_td_peer.head_info head = await peer_chain.get_block_header_by_hash(head_info.block_hash) assert head.block_number == head_info.block_number # In order to answer queries for contract code, geth needs the state trie entry for the block # we specify in the query, but because of fast sync we can only assume it has that for recent # blocks, so we use the current head to lookup the code for the contract below. # https://ropsten.etherscan.io/address/0x95a48dca999c89e4e284930d9b9af973a7481287 contract_addr = decode_hex('0x8B09D9ac6A4F7778fCb22852e879C7F3B2bEeF81') contract_code = await peer_chain.get_contract_code(head.hash, contract_addr) assert encode_hex(contract_code) == '0x600060006000600060006000356000f1' account = await peer_chain.get_account(head.hash, contract_addr) assert account.code_hash == keccak(contract_code) assert account.balance == 0