def _setup_standalone_component(
component_type: Union[Type['TrioIsolatedComponent'],
Type['AsyncioIsolatedComponent']],
app_identifier: str,
) -> Tuple[Union['TrioIsolatedComponent', 'AsyncioIsolatedComponent'], Tuple[
str, ...]]:
if app_identifier == APP_IDENTIFIER_ETH1:
app_cfg: Type[BaseAppConfig] = Eth1AppConfig
elif app_identifier == APP_IDENTIFIER_BEACON:
app_cfg = BeaconAppConfig
else:
raise ValueError("Unknown app identifier: %s", app_identifier)
# Require a root dir to be specified as we don't want to mess with the default one.
for action in parser._actions:
if action.dest == 'trinity_root_dir':
action.required = True
break
component_type.configure_parser(parser, subparser)
parser.add_argument(
'--connect-to-endpoints',
help=
"A list of event bus IPC files for components we should connect to",
nargs='+',
default=tuple(),
)
args = parser.parse_args()
# FIXME: Figure out a way to avoid having to set this.
args.sync_mode = SYNC_FULL
args.enable_metrics = False
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s: %(message)s',
datefmt='%H:%M:%S')
if args.log_levels is not None:
for name, level in args.log_levels.items():
if name is None:
name = ''
get_logger(name).setLevel(level)
trinity_config = TrinityConfig.from_parser_args(args, app_identifier,
(app_cfg, ))
trinity_config.trinity_root_dir.mkdir(exist_ok=True)
if not is_data_dir_initialized(trinity_config):
initialize_data_dir(trinity_config)
boot_info = BootInfo(
args=args,
trinity_config=trinity_config,
min_log_level=None,
logger_levels=None,
profile=False,
)
return component_type(boot_info), args.connect_to_endpoints
class NoopMetricsService(Service, MetricsServiceAPI):
"""
A ``MetricsServiceAPI`` implementation that provides a stub registry where every action results
in a noop. Intended to be used when collecting of metrics is disabled. Every collected metric
will only incur the cost of a noop.
"""
logger = get_logger(
'trinity.components.builtin.metrics.NoopMetricsService')
def __init__(self,
influx_server: str = '',
influx_user: str = '',
influx_password: str = '',
influx_database: str = '',
host: str = '',
reporting_frequency: int = 10):
self._registry = NoopMetricsRegistry()
@property
def registry(self) -> NoopMetricsRegistry:
"""
Return the :class:`trinity.components.builtin.metrics.registry.NoopMetricsRegistry` at which
metrics instruments can be registered and retrieved.
"""
return self._registry
async def run(self) -> None:
self.logger.info("Running NoopMetricsService")
await self.manager.wait_finished()
async def continuously_report(self) -> None:
pass
def test_get_logger():
logger = get_logger("foo.bar.baz")
assert isinstance(logger, eth_utils.ExtendedDebugLogger)
assert logger.parent.name == "foo.bar"
assert isinstance(logger.parent, eth_utils.ExtendedDebugLogger)
assert logger.parent.parent.name == "foo"
assert isinstance(logger.parent.parent, eth_utils.ExtendedDebugLogger)
class TxComponent(AsyncioIsolatedComponent):
name = "TxComponent"
logger = get_logger('trinity.components.tx_pool.TxPool')
@classmethod
def configure_parser(cls, arg_parser: ArgumentParser,
subparser: _SubParsersAction) -> None:
arg_parser.add_argument(
"--disable-tx-pool",
action="store_true",
help="Disables the Transaction Pool",
)
@classmethod
def validate_cli(cls, boot_info: BootInfo) -> None:
network_id = boot_info.trinity_config.network_id
if network_id not in {
MAINNET_NETWORK_ID, ROPSTEN_NETWORK_ID, GOERLI_NETWORK_ID
}:
if not boot_info.args.disable_tx_pool:
raise ValidationError(
"The TxPool component only supports Mainnet, Ropsten and Goerli."
"You can run with the transaction pool disabled using "
"--disable-tx-pool")
@property
def is_enabled(self) -> bool:
light_mode = self._boot_info.args.sync_mode == SYNC_LIGHT
is_disable = self._boot_info.args.disable_tx_pool
is_supported = not light_mode
is_enabled = not is_disable and is_supported
if not is_disable and not is_supported:
self.logger.warning(
"Transaction pool disabled. Not supported in light mode.")
return is_enabled
async def do_run(self, event_bus: EndpointAPI) -> None:
boot_info = self._boot_info
trinity_config = boot_info.trinity_config
db = DBClient.connect(trinity_config.database_ipc_path)
with db:
app_config = trinity_config.get_app_config(Eth1AppConfig)
chain_config = app_config.get_chain_config()
chain = chain_config.full_chain_class(db)
validator = DefaultTransactionValidator.from_network_id(
chain,
boot_info.trinity_config.network_id,
)
proxy_peer_pool = ETHProxyPeerPool(event_bus,
TO_NETWORKING_BROADCAST_CONFIG)
async with background_asyncio_service(proxy_peer_pool):
tx_pool = TxPool(event_bus, proxy_peer_pool, validator)
async with background_asyncio_service(tx_pool) as manager:
await manager.wait_finished()
def __init__(self, chain: AsyncChainAPI, chaindb: BaseAsyncChainDB,
base_db: AtomicDatabaseAPI, peer_pool: ETHPeerPool) -> None:
self.logger = get_logger('trinity.sync.full.FullChainSyncer')
self.chain = chain
self.chaindb = chaindb
self.base_db = base_db
self.peer_pool = peer_pool
class AsyncioComponentForTest(AsyncioIsolatedComponent):
name = "component-test-asyncio"
endpoint_name = 'component-test-asyncio'
logger = get_logger('trinity.testing.AsyncioComponentForTest')
def get_subprocess_kwargs(self) -> SubprocessKwargs:
# This is needed so that pytest can capture the subproc's output. Otherwise it will crash
# with a "io.UnsupportedOperation: redirected stdin is pseudofile, has no fileno()" error.
return merge(
super().get_subprocess_kwargs(), {
'stdin': subprocess.PIPE,
'stdout': subprocess.PIPE,
'stderr': subprocess.PIPE,
})
@property
def is_enabled(self) -> bool:
return True
async def do_run(self, event_bus: EndpointAPI) -> None:
self.logger.info('Entered `do_run`')
service = ComponentTestService(event_bus)
try:
async with background_asyncio_service(service) as manager:
self.logger.info('Running service')
try:
await manager.wait_finished()
finally:
self.logger.info('Exiting `do_run`')
finally:
# XXX: We never reach this line, so if you run test_isolated_component.py by itself it
# will pass but hang forever after pytest reports success.
# Figuring this out is probably the key to fixing our shutdown.
self.logger.info('Finished: `do_run`')
class PausingVM(original_vm_class): # type: ignore
logger = get_logger(f'eth.vm.base.VM.{original_vm_class.__name__}')
last_log_time = 0.0
@classmethod
def get_state_class(cls) -> Type[StateAPI]:
return PausingVMState
def get_beam_stats(self) -> BeamStats:
return self.state.stats_counter
def transaction_applied_hook(
self, transaction_index: int,
transactions: Sequence[SignedTransactionAPI],
base_header: BlockHeaderAPI, partial_header: BlockHeaderAPI,
computation: ComputationAPI, receipt: ReceiptAPI) -> None:
num_transactions = len(transactions)
now = time.monotonic()
if urgent:
# The currently-importing block
if transaction_index == num_transactions - 1:
logger = self.logger.info
log_header = "Beamed"
elif now - self.last_log_time > MIN_GAS_LOG_WAIT:
logger = self.logger.info
log_header = "Beaming"
else:
# Logged an update too recently, skipping...
return
else:
# Don't log anything for preview executions, for now
return
beam_stats = self.get_beam_stats()
fraction_complete = partial_header.gas_used / base_header.gas_used
if fraction_complete:
total_est = beam_stats.data_pause_time / fraction_complete
est_time = humanize_seconds(total_est -
beam_stats.data_pause_time)
else:
est_time = "?"
logger(
"%s: #%d txn %d/%d, rtt: %.3fs, wait: %s, nodes: %d, gas: %s/%s (%.1f%%) ETA %s",
log_header,
base_header.block_number,
transaction_index + 1,
num_transactions,
beam_stats.avg_rtt,
humanize_seconds(beam_stats.data_pause_time),
beam_stats.num_nodes,
f"{partial_header.gas_used:,d}",
f"{base_header.gas_used:,d}",
100 * fraction_complete,
est_time,
)
self.last_log_time = now
def __init__(self, headerdb: BaseAsyncHeaderDB,
peer_pool: LESPeerPool) -> None:
PeerSubscriber.__init__(self)
self.logger = get_logger('trinity.sync.light.LightPeerChain')
self.headerdb = headerdb
self.peer_pool = peer_pool
self._pending_replies = weakref.WeakValueDictionary()
def __init__(self, db: AtomicDatabaseAPI, peer_pool: ETHPeerPool,
queen_tracker: QueenTrackerAPI,
event_bus: EndpointAPI) -> None:
self.logger = get_logger('trinity.sync.beam.BeamDownloader')
self._db = db
self._trie_db = HexaryTrie(db)
self._event_bus = event_bus
# Track the needed node data that is urgent and important:
buffer_size = MAX_STATE_FETCH * REQUEST_BUFFER_MULTIPLIER
self._node_tasks = TaskQueue[Hash32](buffer_size, lambda task: 0)
# list of events waiting on new data
self._new_data_event: asyncio.Event = asyncio.Event()
self._preview_events = {}
self._peer_pool = peer_pool
# Track node data for upcoming blocks
self._block_number_lookup = defaultdict(lambda: BlockNumber(0))
self._maybe_useful_nodes = TaskQueue[Hash32](
buffer_size,
# Prefer trie nodes from earliest blocks
lambda node_hash: self._block_number_lookup[node_hash],
)
self._num_urgent_requests_by_peer = Counter()
self._num_predictive_requests_by_peer = Counter()
self._queen_tracker = queen_tracker
self._threadpool = ThreadPoolExecutor()
asyncio.get_event_loop().set_default_executor(self._threadpool)
def __init__(self, state_downloader: BeamDownloader,
event_bus: EndpointAPI) -> None:
self.logger = get_logger(
'trinity.sync.beam.chain.MissingDataEventHandler')
self._state_downloader = state_downloader
self._event_bus = event_bus
self._minimum_beam_block_number = 0
async def run(self) -> None:
"""
Call chain is:
- multiprocessing.Process -> _run_process
* isolates to a new process
- _run_process -> run_process
* sets up subprocess logging
- run_process -> _do_run
* runs the event loop and transitions into async context
- _do_run -> do_run
* sets up event bus and then enters user function.
"""
process = ctx.Process(
target=self.run_process,
args=(self._boot_info, ),
)
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, process.start)
try:
await loop.run_in_executor(None, process.join)
finally:
kill_process_gracefully(
process,
get_logger('trinity.extensibility.TrioIsolatedComponent'),
)
def __init__(self, db: AtomicDatabaseAPI, peer_pool: ETHPeerPool,
queen_tracker: QueenTrackerAPI,
event_bus: EndpointAPI) -> None:
self.logger = get_logger('trinity.sync.beam.BeamDownloader')
self._db = db
self._trie_db = HexaryTrie(db)
self._event_bus = event_bus
# Track the needed node data that is urgent and important:
buffer_size = MAX_STATE_FETCH * REQUEST_BUFFER_MULTIPLIER
self._node_tasks = TaskQueue[Hash32](buffer_size, lambda task: 0)
# list of events waiting on new data
self._new_data_events: Set[asyncio.Event] = set()
self._peer_pool = peer_pool
# Track node data for upcoming blocks
self._maybe_useful_nodes = TaskQueue[Hash32](
buffer_size,
# Everything is the same priority, for now
lambda node_hash: 0,
)
self._num_urgent_requests_by_peer = Counter()
self._num_predictive_requests_by_peer = Counter()
self._queen_tracker = queen_tracker
def __init__(self,
chain: AsyncChainAPI,
db: BaseAsyncChainDB,
peer_pool: ETHPeerPool,
enable_backfill: bool = True,
checkpoint: Checkpoint = None) -> None:
self.logger = get_logger('trinity.sync.header.chain.HeaderChainSyncer')
self._db = db
self._checkpoint = checkpoint
self._enable_backfill = enable_backfill
self._chain = chain
self._peer_pool = peer_pool
if checkpoint is None:
self._launch_strategy: SyncLaunchStrategyAPI = FromGenesisLaunchStrategy(
db)
else:
self._launch_strategy = FromCheckpointLaunchStrategy(
db,
chain,
checkpoint,
peer_pool,
)
self._header_syncer = ETHHeaderChainSyncer(chain, db, peer_pool,
self._launch_strategy)
def __init__(self, db: AtomicDatabaseAPI, peer_pool: ETHPeerPool) -> None:
self.logger = get_logger(
'trinity.sync.beam.backfill.BeamStateBackfill')
self._db = db
self._peer_pool = peer_pool
self._is_missing: Set[Hash32] = set()
self._num_requests_by_peer = Counter()
self._queening_queue = QueeningQueue(peer_pool)
# Track the nodes that we are requesting in the account trie
self._account_tracker = TrieNodeRequestTracker()
self._storage_trackers: Dict[Hash32, TrieNodeRequestTracker] = {}
self._bytecode_trackers: Dict[Hash32, TrieNodeRequestTracker] = {}
# The most recent root hash to use to navigate the trie
self._next_trie_root_hash: Optional[Hash32] = None
self._begin_backfill = asyncio.Event()
# Only acquire peasant peers for backfill if there are no other coros
# waiting for a peasant. Any other waiter is assumed to be higher priority.
self._external_peasant_usage = SilenceObserver(
minimum_silence_duration=GAP_BETWEEN_TESTS)
def __init__(self, peer_pool: ETHPeerPool) -> None:
self.logger = get_logger('trinity.sync.beam.queen.QueeningQueue')
self._peer_pool = peer_pool
self._knights = WaitingPeers(NodeDataV65)
self._peasants = WaitingPeers(NodeDataV65)
self._queen_updated = asyncio.Event()
self._desired_knights = 0
self._num_peers = 0
def __init__(self, chain: AsyncChainAPI, db: BaseAsyncChainDB,
peer_pool: ETHPeerPool) -> None:
self.logger = get_logger(
'trinity.sync.header.chain.SequentialHeaderChainGapSyncer')
self._chain = chain
self._db = db
self._peer_pool = peer_pool
def __init__(self, event_bus: EndpointAPI, preferred_nodes: List[NodeAPI],
max_peers: int) -> None:
self.event_bus = event_bus
self.max_peers = max_peers
self.preferred_nodes = preferred_nodes
self._preferred_node_tracker = {}
self.logger = get_logger(
'trinity.components.preferred_node.PreferredNodeService')
def __init__(self, chain: AsyncChainAPI, db: BaseAsyncHeaderDB,
peer: BaseChainPeer) -> None:
self.logger = get_logger('trinity.sync.common.chain.PeerHeaderSyncer')
self.chain = chain
self.db = db
self.sync_progress: SyncProgress = None
self._peer = peer
self._target_header_hash = peer.head_info.head_hash
class PausingVM(original_vm_class): # type: ignore
logger = get_logger(f'eth.vm.base.VM.{original_vm_class.__name__}')
@classmethod
def get_state_class(cls) -> Type[StateAPI]:
return PausingVMState
def get_beam_stats(self) -> BeamStats:
return self.state.stats_counter
class BlockImportServer(Service):
logger = get_logger('trinity.sync.beam.BlockImportServer')
def __init__(
self,
event_bus: EndpointAPI,
beam_chain: BeamChain) -> None:
self._event_bus = event_bus
self._beam_chain = beam_chain
async def run(self) -> None:
self.manager.run_daemon_task(self.serve, self._event_bus, self._beam_chain)
await self.manager.wait_finished()
async def serve(
self,
event_bus: EndpointAPI,
beam_chain: BeamChain) -> None:
"""
Listen to DoStatelessBlockImport events, and import block when received.
Reply with StatelessBlockImportDone when import is complete.
"""
loop = asyncio.get_event_loop()
async for event in event_bus.stream(DoStatelessBlockImport):
# launch in new thread, so we don't block the event loop!
import_completion = loop.run_in_executor(
# Maybe build the pausing chain inside the new process?
None,
partial_import_block(beam_chain, event.block),
)
# Wrapped in `asyncio.shield` because we want to hang the service from
# shutting down until block import is complete.
# In the tests, for example, we await cancel() this service, so that we know
# that the in-progress block is complete. Then below, we do not send back
# the import completion (so the import server won't get triggered again).
try:
await asyncio.shield(import_completion)
except StateUnretrievable as exc:
self.logger.debug(
"Not broadcasting about %s Beam import. Listening for next request, because %r",
event.block,
exc
)
else:
if self.manager.is_running:
_broadcast_import_complete(
event_bus,
event.block,
event.broadcast_config(),
import_completion, # type: ignore
)
if not self.manager.is_running:
break
def __init__(self, chain: AsyncChainAPI, db: BaseAsyncChainDB,
peer_pool: ETHPeerPool) -> None:
self.logger = get_logger(
'trinity.sync.header.chain.SequentialHeaderChainGapSyncer')
self._chain = chain
self._db = db
self._peer_pool = peer_pool
self._pauser = Pauser()
self._max_backfill_header_at_once = MAX_BACKFILL_HEADERS_AT_ONCE
def __init__(self,
header_syncer: ETHHeaderChainSyncer,
db: BaseAsyncHeaderDB,
force_end_block_number: int = None,
launch_strategy: SyncLaunchStrategyAPI = None) -> None:
self.logger = get_logger('trinity.sync.beam.chain.HeaderOnlyPersist')
self._db = db
self._header_syncer = header_syncer
self._final_headers: Tuple[BlockHeaderAPI, ...] = None
self._force_end_block_number = force_end_block_number
self._launch_strategy = launch_strategy
def __init__(self,
chain: AsyncChainAPI,
db: BaseAsyncChainDB,
peer_pool: ETHPeerPool,
max_headers: int = None) -> None:
self.logger = get_logger(
'trinity.sync.header.chain.HeaderChainGapSyncer')
self._chain = chain
self._db = db
self._peer_pool = peer_pool
self._max_headers = max_headers
def __init__(self,
handler: Callable[..., Any],
host: str = '127.0.0.1',
port: int = 8545) -> None:
self.host = host
self.port = port
self.server = web.Server(handler)
self.logger = get_logger('trinity.http.HTTPServer')
# aiohttp logs every HTTP request as INFO so we want to reduce the general log level for
# this particular logger to WARNING except if the Trinity is configured to write DEBUG2 logs
if logging.getLogger().level != DEBUG2_LEVEL_NUM:
logging.getLogger('aiohttp.access').setLevel(logging.WARNING)
def __init__(self, node_hashes: Sequence[Hash32]) -> None:
self.logger = get_logger(
"trinity.protocol.eth.validators.GetNodeDataValidator")
self.node_hashes = node_hashes
# Check for uniqueness
num_requested = len(node_hashes)
num_unique = len(set(node_hashes))
if num_requested != num_unique:
self.logger.warning(
"GetNodeData: Asked peer for %d trie nodes, but %d were duplicates",
num_requested,
num_requested - num_unique,
)
def __init__(self, db: AtomicDatabaseAPI, peer_pool: ETHPeerPool) -> None:
self.logger = get_logger('trinity.sync.beam.backfill.BeamStateBackfill')
self._db = db
# Pending nodes to download
self._node_hashes: List[Hash32] = []
self._peer_pool = peer_pool
self._is_missing: Set[Hash32] = set()
self._num_requests_by_peer = Counter()
self._queening_queue = QueeningQueue(peer_pool)
def __init__(self,
chain: AsyncChainAPI,
db: BaseAsyncHeaderDB,
peer: TChainPeer,
launch_strategy: SyncLaunchStrategyAPI = None) -> None:
self.logger = get_logger('trinity.sync.common.headers.SkeletonSyncer')
self._chain = chain
self._db = db
if launch_strategy is None:
launch_strategy = FromGenesisLaunchStrategy(db, chain)
self._launch_strategy = launch_strategy
self.peer = peer
max_pending_headers = peer.max_headers_fetch * 8
self._fetched_headers = asyncio.Queue(max_pending_headers)
def __init__(self,
chain: AsyncChainAPI,
chaindb: BaseAsyncChainDB,
base_db: AtomicDatabaseAPI,
peer_pool: ETHPeerPool,
event_bus: EndpointAPI,
checkpoint: Checkpoint = None,
force_beam_block_number: BlockNumber = None) -> None:
self.logger = get_logger('trinity.sync.beam.service.BeamSyncService')
self.chain = chain
self.chaindb = chaindb
self.base_db = base_db
self.peer_pool = peer_pool
self.event_bus = event_bus
self.checkpoint = checkpoint
self.force_beam_block_number = force_beam_block_number
class AsyncioIsolatedComponent(BaseIsolatedComponent):
logger = get_logger('trinity.extensibility.asyncio.AsyncioIsolatedComponent')
async def _run_in_process(
self,
async_fn: Callable[..., TReturn],
*args: Any,
subprocess_kwargs: 'SubprocessKwargs' = None,
) -> TReturn:
return await run_in_process(async_fn, *args, subprocess_kwargs=subprocess_kwargs)
async def _do_run(self) -> None:
with child_process_logging(self._boot_info):
endpoint_name = self.get_endpoint_name()
event_bus_service = AsyncioEventBusService(
self._boot_info.trinity_config,
endpoint_name,
)
async with background_asyncio_service(event_bus_service):
event_bus = await event_bus_service.get_event_bus()
try:
if self._boot_info.profile:
with profiler(f'profile_{self.get_endpoint_name()}'):
await self.do_run(event_bus)
else:
# XXX: When open_in_process() injects a KeyboardInterrupt into us (via
# coro.throw()), we hang forever here, until open_in_process() times out
# and sends us a SIGTERM, at which point we exit without executing either
# the except or the finally blocks below.
# See https://github.com/ethereum/trinity/issues/1711 for more.
await self.do_run(event_bus)
except KeyboardInterrupt:
# Currently we never reach this code path, but when we fix the issue above it
# will be needed.
return
finally:
# Once we start seeing this in the logs after a Ctrl-C, we'll likely have
# figured out the issue above.
self.logger.debug("%s: do_run() finished", self)
@abstractmethod
async def do_run(self, event_bus: EndpointAPI) -> None:
"""
Define the entry point of the component. Should be overwritten in subclasses.
"""
...
class ConnectionTrackerServer(Service):
"""
Server to handle the event bus communication for BlacklistEvent and
GetBlacklistedPeersRequest/Response events
"""
logger = get_logger(
'trinity.components.network_db.ConnectionTrackerServer')
def __init__(self, event_bus: EndpointAPI,
tracker: BaseConnectionTracker) -> None:
self.tracker = tracker
self.event_bus = event_bus
async def handle_get_blacklisted_requests(self) -> None:
async for req in self.event_bus.stream(GetBlacklistedPeersRequest):
self.logger.debug2('Received get_blacklisted request')
blacklisted = await self.tracker.get_blacklisted()
await self.event_bus.broadcast(
GetBlacklistedPeersResponse(blacklisted),
req.broadcast_config())
async def handle_blacklist_command(self) -> None:
async for command in self.event_bus.stream(BlacklistEvent):
self.logger.debug2(
'Received blacklist commmand: remote: %s | timeout: %s | reason: %s',
command.remote,
humanize_seconds(command.timeout_seconds),
command.reason,
)
self.tracker.record_blacklist(command.remote,
command.timeout_seconds,
command.reason)
async def run(self) -> None:
self.logger.debug("Running ConnectionTrackerServer")
self.manager.run_daemon_task(
self.handle_blacklist_command,
name='ConnectionTrackerServer.handle_blacklist_command',
)
self.manager.run_daemon_task(
self.handle_get_blacklisted_requests,
name='ConnectionTrackerServer.handle_get_blacklisted_requests,',
)
await self.manager.wait_finished()
