def _test():
import argparse
from evm.p2p import ecies
from evm.chains.ropsten import RopstenChain, ROPSTEN_GENESIS_HEADER
from evm.db.backends.level import LevelDB
from evm.db.backends.memory import MemoryDB
logging.basicConfig(level=logging.INFO,
format='%(levelname)s: %(message)s')
parser = argparse.ArgumentParser()
parser.add_argument('-db', type=str, required=True)
parser.add_argument('-root-hash',
type=str,
required=True,
help='Hex encoded root hash')
args = parser.parse_args()
chaindb = BaseChainDB(MemoryDB())
chaindb.persist_header_to_db(ROPSTEN_GENESIS_HEADER)
peer_pool = PeerPool(ETHPeer, chaindb, RopstenChain.network_id,
ecies.generate_privkey())
asyncio.ensure_future(peer_pool.run())
state_db = LevelDB(args.db)
root_hash = decode_hex(args.root_hash)
downloader = StateDownloader(state_db, root_hash, peer_pool)
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(downloader.run())
except KeyboardInterrupt:
pass
loop.run_until_complete(downloader.stop())
loop.run_until_complete(peer_pool.stop())
loop.close()
def _test():
import argparse
import signal
from evm.p2p import ecies
from evm.chains.ropsten import RopstenChain, ROPSTEN_GENESIS_HEADER
from evm.db.backends.level import LevelDB
from evm.db.backends.memory import MemoryDB
logging.basicConfig(level=logging.INFO,
format='%(levelname)s: %(message)s')
parser = argparse.ArgumentParser()
parser.add_argument('-db', type=str, required=True)
parser.add_argument('-root-hash',
type=str,
required=True,
help='Hex encoded root hash')
args = parser.parse_args()
chaindb = ChainDB(MemoryDB())
chaindb.persist_header_to_db(ROPSTEN_GENESIS_HEADER)
peer_pool = PeerPool(ETHPeer, chaindb, RopstenChain.network_id,
ecies.generate_privkey())
asyncio.ensure_future(peer_pool.run())
state_db = LevelDB(args.db)
root_hash = decode_hex(args.root_hash)
downloader = StateDownloader(state_db, root_hash, peer_pool)
loop = asyncio.get_event_loop()
for sig in [signal.SIGINT, signal.SIGTERM]:
loop.add_signal_handler(sig, downloader.cancel_token.trigger)
async def run():
# downloader.run() will run in a loop until the SIGINT/SIGTERM handler triggers its cancel
# token, at which point it returns and we stop the pool and downloader.
await downloader.run()
await peer_pool.stop()
await downloader.stop()
loop.run_until_complete(run())
loop.close()
def run_networking_process(chain_config, sync_mode):
class DBManager(BaseManager):
pass
DBManager.register('get_db', proxytype=DBProxy)
DBManager.register('get_chaindb', proxytype=ChainDBProxy)
manager = DBManager(address=chain_config.database_ipc_path)
manager.connect()
chaindb = manager.get_chaindb()
if not is_data_dir_initialized(chain_config):
# TODO: this will only work as is for chains with known genesis
# parameters. Need to flesh out how genesis parameters for custom
# chains are defined and passed around.
initialize_data_dir(chain_config)
if not is_database_initialized(chaindb):
initialize_database(chain_config, chaindb)
chain_class = get_chain_protocol_class(chain_config, sync_mode=sync_mode)
peer_pool = PeerPool(LESPeer, chaindb, chain_config.network_id,
chain_config.nodekey)
async def run():
try:
asyncio.ensure_future(peer_pool.run())
# chain.run() will run in a loop until our atexit handler is called, at which point it returns
# and we cleanly stop the pool and chain.
await chain.run()
finally:
await peer_pool.stop()
await chain.stop()
chain = chain_class(chaindb, peer_pool)
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(run())
except KeyboardInterrupt:
pass
def cleanup():
# This is to instruct chain.run() to exit, which will cause the event loop to stop.
chain._should_stop.set()
loop.close()
atexit.register(cleanup)
def __init__(self, chaindb, state_db, root_hash, network_id, privkey):
self.peer_pool = PeerPool(ETHPeer, chaindb, network_id, privkey, self.msg_handler)
self.root_hash = root_hash
self.scheduler = StateSync(root_hash, state_db, self.logger)
class StateDownloader:
logger = logging.getLogger("evm.p2p.state.StateDownloader")
_pending_nodes = {} # type: Dict[Any, float]
_total_processed_nodes = 0
_report_interval = 10 # Number of seconds between progress reports.
# TODO: Experiment with different timeout/max_pending values to find the combination that
# yields the best results.
# FIXME: Should use the # of peers times MAX_STATE_FETCH here
_max_pending = 5 * MAX_STATE_FETCH
_reply_timeout = 10 # seconds
# For simplicity/readability we use 0 here to force a report on the first iteration of the
# loop.
_last_report_time = 0
def __init__(self, chaindb, state_db, root_hash, network_id, privkey):
self.peer_pool = PeerPool(ETHPeer, chaindb, network_id, privkey, self.msg_handler)
self.root_hash = root_hash
self.scheduler = StateSync(root_hash, state_db, self.logger)
def msg_handler(self, peer: BasePeer, cmd: protocol.Command,
msg: protocol._DecodedMsgType) -> None:
"""The callback passed to BasePeer, called for every incoming message."""
peer = cast(ETHPeer, peer)
if isinstance(cmd, eth.NodeData):
self.logger.debug("Processing NodeData with %d entries" % len(msg))
for node in msg:
self._total_processed_nodes += 1
node_key = keccak(node)
try:
self.scheduler.process([(node_key, node)])
except SyncRequestAlreadyProcessed:
# This means we received a node more than once, which can happen when we retry
# after a timeout.
pass
# A node may be received more than once, so pop() with a default value.
self._pending_nodes.pop(node_key, None)
# FIXME: Need a better criteria to select peers here.
async def get_random_peer(self) -> ETHPeer:
while len(self.peer_pool.peers) == 0:
self.logger.debug("No connected peers, sleeping a bit")
await asyncio.sleep(0.5)
peer = random.choice(self.peer_pool.peers)
return cast(ETHPeer, peer)
async def stop(self):
await self.peer_pool.stop()
async def request_next_batch(self):
requests = self.scheduler.next_batch(MAX_STATE_FETCH)
if not len(requests):
# Although our run() loop frequently yields control to let our msg handler process
# received nodes (scheduling new requests), there may be cases when the pending nodes
# take a while to arrive thus causing the scheduler to run out of new requests for a
# while.
self.logger.debug("Scheduler queue is empty, not requesting any nodes")
return
self.logger.debug("Requesting %d trie nodes" % len(requests))
await self.request_nodes([request.node_key for request in requests])
async def request_nodes(self, node_keys):
peer = await self.get_random_peer()
now = time.time()
for node_key in node_keys:
self._pending_nodes[node_key] = now
peer.sub_proto.send_get_node_data(node_keys)
async def retry_timedout(self):
timed_out = []
now = time.time()
for node_key, req_time in list(self._pending_nodes.items()):
if now - req_time > self._reply_timeout:
timed_out.append(node_key)
if len(timed_out) == 0:
return
self.logger.debug("Re-requesting %d trie nodes" % len(timed_out))
await self.request_nodes(timed_out)
async def run(self):
asyncio.ensure_future(self.peer_pool.run())
self.logger.info("Starting state sync for root hash %s" % encode_hex(self.root_hash))
while self.scheduler.has_pending_requests:
# Request new nodes if we haven't reached the limit of pending nodes.
if len(self._pending_nodes) < self._max_pending:
await self.request_next_batch()
# Retry pending nodes that timed out.
if len(self._pending_nodes):
await self.retry_timedout()
if len(self._pending_nodes) > self._max_pending:
# Slow down if we've reached the limit of pending nodes.
self.logger.debug("Pending trie nodes limit reached, sleeping a bit")
await asyncio.sleep(0.3)
else:
# Yield control to ensure the Peer's msg_handler callback is called to process any
# nodes we may have received already. Otherwise we spin too fast and don't process
# received nodes often enough.
await asyncio.sleep(0)
self._maybe_report_progress()
self.logger.info("Finished state sync with root hash %s" % encode_hex(self.root_hash))
def _maybe_report_progress(self):
if (time.time() - self._last_report_time) >= self._report_interval:
self._last_report_time = time.time()
self.logger.info("Nodes processed: %d" % self._total_processed_nodes)
self.logger.info(
"Nodes requested but not received yet: %d" % len(self._pending_nodes))
self.logger.info(
"Nodes scheduled but not requested yet: %d" % len(self.scheduler.requests))
parser.add_argument('-debug', action='store_true')
args = parser.parse_args()
print("Logging to", LOGFILE)
if args.debug:
LOGLEVEL = logging.DEBUG
logging.basicConfig(level=LOGLEVEL, filename=LOGFILE)
DemoLightChain = LightChain.configure(
name='Demo LightChain',
vm_configuration=MAINNET_VM_CONFIGURATION,
network_id=ROPSTEN_NETWORK_ID,
)
chaindb = ChainDB(LevelDB(args.db))
peer_pool = PeerPool(LESPeer, chaindb, ROPSTEN_NETWORK_ID,
ecies.generate_privkey())
try:
chaindb.get_canonical_head()
except CanonicalHeadNotFound:
# We're starting with a fresh DB.
chain = DemoLightChain.from_genesis_header(chaindb, ROPSTEN_GENESIS_HEADER,
peer_pool)
else:
# We're reusing an existing db.
chain = DemoLightChain(chaindb, peer_pool)
async def run():
asyncio.ensure_future(peer_pool.run())
# chain.run() will run in a loop until our atexit handler is called, at which point it returns
# and we cleanly stop the pool and chain.
def _test():
import argparse
import signal
from evm.chains.mainnet import (MAINNET_GENESIS_HEADER,
MAINNET_VM_CONFIGURATION,
MAINNET_NETWORK_ID)
from evm.chains.ropsten import ROPSTEN_GENESIS_HEADER, ROPSTEN_NETWORK_ID
from evm.db.backends.level import LevelDB
from evm.exceptions import CanonicalHeadNotFound
from evm.p2p import ecies
from evm.p2p.integration_test_helpers import LocalGethPeerPool
logging.basicConfig(level=logging.INFO,
format='%(levelname)s: %(message)s')
logging.getLogger("evm.p2p.lightchain.LightChain").setLevel(logging.DEBUG)
parser = argparse.ArgumentParser()
parser.add_argument('-db', type=str, required=True)
parser.add_argument('-mainnet', action="store_true")
parser.add_argument('-local-geth', action="store_true")
args = parser.parse_args()
GENESIS_HEADER = ROPSTEN_GENESIS_HEADER
NETWORK_ID = ROPSTEN_NETWORK_ID
if args.mainnet:
GENESIS_HEADER = MAINNET_GENESIS_HEADER
NETWORK_ID = MAINNET_NETWORK_ID
DemoLightChain = LightChain.configure(
'DemoLightChain',
vm_configuration=MAINNET_VM_CONFIGURATION,
network_id=NETWORK_ID,
)
chaindb = BaseChainDB(LevelDB(args.db))
if args.local_geth:
peer_pool = LocalGethPeerPool(LESPeer, chaindb, NETWORK_ID,
ecies.generate_privkey())
else:
peer_pool = PeerPool(LESPeer, chaindb, NETWORK_ID,
ecies.generate_privkey())
try:
chaindb.get_canonical_head()
except CanonicalHeadNotFound:
# We're starting with a fresh DB.
chain = DemoLightChain.from_genesis_header(chaindb, GENESIS_HEADER,
peer_pool)
else:
# We're reusing an existing db.
chain = DemoLightChain(chaindb, peer_pool)
asyncio.ensure_future(peer_pool.run())
loop = asyncio.get_event_loop()
async def run():
# chain.run() will run in a loop until stop() (registered as SIGINT/SIGTERM handler) is
# called, at which point it returns and we cleanly stop the pool and chain.
await chain.run()
await peer_pool.stop()
await chain.stop()
def stop():
chain._should_stop.set()
for sig in [signal.SIGINT, signal.SIGTERM]:
loop.add_signal_handler(sig, stop)
loop.run_until_complete(run())
loop.close()
def main():
args = parser.parse_args()
if args.ropsten:
chain_identifier = ROPSTEN
else:
# TODO: mainnet
chain_identifier = ROPSTEN
if args.light:
sync_mode = SYNC_LIGHT
else:
# TODO: actually use args.sync_mode (--sync-mode)
sync_mode = SYNC_LIGHT
chain_config = ChainConfig.from_parser_args(
chain_identifier,
args,
)
# if console command, run the trinity CLI
if args.subcommand == 'console':
use_ipython = not args.vanilla_shell
debug = args.log_level.upper() == 'DEBUG'
# TODO: this should use the base `Chain` class rather than the protocol
# class since it's just a repl with access to the chain.
chain_class = get_chain_protocol_class(chain_config, sync_mode)
chaindb = ChainDB(LevelDB(chain_config.database_dir))
peer_pool = PeerPool(LESPeer, chaindb, chain_config.network_id, chain_config.nodekey)
chain = chain_class(chaindb, peer_pool)
args.func(chain, use_ipython=use_ipython, debug=debug)
sys.exit(0)
logger, log_queue, listener = setup_trinity_logging(args.log_level.upper())
# start the listener thread to handle logs produced by other processes in
# the local logger.
listener.start()
# First initialize the database process.
database_server_process = ctx.Process(
target=run_database_process,
args=(
chain_config,
LevelDB,
),
kwargs={'log_queue': log_queue}
)
# For now we just run the light sync against ropsten by default.
networking_process = ctx.Process(
target=run_networking_process,
args=(chain_config, sync_mode),
kwargs={'log_queue': log_queue}
)
# start the processes
database_server_process.start()
wait_for_ipc(chain_config.database_ipc_path)
networking_process.start()
try:
networking_process.join()
except KeyboardInterrupt:
logger.info('Keyboard Interrupt: Stopping')
kill_process_gracefully(networking_process)
logger.info('KILLED networking_process')
kill_process_gracefully(database_server_process)
logger.info('KILLED database_server_process')