def __init__(self, chain: AsyncChainAPI, db: BaseAsyncChainDB,
peer_pool: ETHPeerPool) -> None:
self._chain = chain
self._db = db
self._peer_pool = peer_pool
self._pauser = Pauser()
self._body_syncer: FastChainBodySyncer = None
def __init__(self, chain: AsyncChainAPI, db: BaseAsyncChainDB,
peer_pool: ETHPeerPool) -> None:
self._chain = chain
self._db = db
self._peer_pool = peer_pool
self._pauser = Pauser()
self._body_syncer: FastChainBodySyncer = None
self._max_backfill_block_bodies_at_once = MAX_BACKFILL_BLOCK_BODIES_AT_ONCE
def __init__(self, chain: AsyncChainAPI, db: BaseAsyncChainDB,
peer_pool: ETHPeerPool) -> None:
self.logger = get_logger(
'trinity.sync.header.chain.SequentialHeaderChainGapSyncer')
self._chain = chain
self._db = db
self._peer_pool = peer_pool
self._pauser = Pauser()
self._max_backfill_header_at_once = MAX_BACKFILL_HEADERS_AT_ONCE
class SequentialHeaderChainGapSyncer(Service):
def __init__(self, chain: AsyncChainAPI, db: BaseAsyncChainDB,
peer_pool: ETHPeerPool) -> None:
self.logger = get_logger(
'trinity.sync.header.chain.SequentialHeaderChainGapSyncer')
self._chain = chain
self._db = db
self._peer_pool = peer_pool
self._pauser = Pauser()
self._max_backfill_header_at_once = MAX_BACKFILL_HEADERS_AT_ONCE
def pause(self) -> None:
"""
Pause the sync after the current operation has finished.
"""
# We just switch the toggle but let the sync finish the current segment. It will wait for
# the resume call before it starts a new segment.
self._pauser.pause()
self.logger.debug2(
"Pausing SequentialHeaderChainGapSyncer after current operation finishs"
)
def resume(self) -> None:
"""
Resume the sync.
"""
self._pauser.resume()
self.logger.debug2("SequentialHeaderChainGapSyncer resumed")
async def run(self) -> None:
while self.manager.is_running:
if self._pauser.is_paused:
await self._pauser.await_resume()
gaps, _ = self._db.get_header_chain_gaps()
if len(gaps) < 1:
self.logger.info("No more gaps to fill. Exiting")
self.manager.cancel()
return
else:
self.logger.debug(f"Starting gap sync at {gaps[0]}")
syncer = HeaderChainGapSyncer(
self._chain,
self._db,
self._peer_pool,
max_headers=self._max_backfill_header_at_once,
)
async with background_asyncio_service(syncer) as manager:
await manager.wait_finished()
class BodyChainGapSyncer(Service):
"""
A service to sync historical blocks without executing them. This service is meant to be run
in tandem with other operations that sync the state.
"""
_idle_time = BLOCK_BACKFILL_IDLE_TIME
_starting_tip: BlockHeaderAPI = None
logger = get_logger('trinity.sync.beam.chain.BodyChainGapSyncer')
def __init__(self,
chain: AsyncChainAPI,
db: BaseAsyncChainDB,
peer_pool: ETHPeerPool) -> None:
self._chain = chain
self._db = db
self._peer_pool = peer_pool
self._pauser = Pauser()
self._body_syncer: FastChainBodySyncer = None
self._max_backfill_block_bodies_at_once = MAX_BACKFILL_BLOCK_BODIES_AT_ONCE
async def _setup_for_next_gap(self) -> None:
gap_start, gap_end = self._get_next_gap()
fill_start = BlockNumber(max(
gap_start,
gap_end - self._max_backfill_block_bodies_at_once,
))
start_num = BlockNumber(fill_start - 1)
_starting_tip = await self._db.coro_get_canonical_block_header_by_number(start_num)
if self._pauser.is_paused:
# If the syncer was paused while we were busy setting it up throw the current setup
# away. A new setup will be performed as soon as `resume()` was called again.
raise ValidationError("Syncer was paused by the user")
async def _get_launch_header() -> BlockHeaderAPI:
return _starting_tip
self.logger.debug("Starting to sync missing blocks from #%s to #%s", fill_start, gap_end)
self._body_syncer = FastChainBodySyncer(
self._chain,
self._db,
self._peer_pool,
DatabaseBlockRangeHeaderSyncer(self._db, (fill_start, gap_end,)),
launch_header_fn=_get_launch_header,
should_skip_header_fn=body_for_header_exists(self._db, self._chain)
)
self._body_syncer.logger = self.logger
def _get_next_gap(self) -> BlockRange:
gaps, future_tip_block = self._db.get_chain_gaps()
header_gaps, future_tip_header = self._db.get_header_chain_gaps()
try:
actionable_gap = self.get_topmost_actionable_gap(gaps, header_gaps)
except NoActionableGap:
# We do not have gaps in the chain of blocks but we may still have a gap from the last
# block up until the highest consecutive written header.
if len(header_gaps) > 0:
# The header chain has gaps, find out the lowest missing header
lowest_missing_header, _ = header_gaps[0]
else:
# It doesn't have gaps, so the future_tip_header is the lowest missing header
lowest_missing_header = future_tip_header
highest_consecutive_header = lowest_missing_header - 1
if highest_consecutive_header >= future_tip_block:
# The header before the lowest missing header is the highest consecutive header
# that exists in the db and it is higher than the future tip block. That's a gap
# we can try to close.
return future_tip_block, BlockNumber(highest_consecutive_header)
else:
raise ValidationError("No gaps in the chain of blocks")
else:
return actionable_gap
def get_topmost_actionable_gap(self,
gaps: Tuple[BlockRange, ...],
header_gaps: Tuple[BlockRange, ...]) -> BlockRange:
'''
Returns the most recent gap of blocks of max size = _max_backfill_block_bodies_at_once
for which the headers exist in DB, along with the header preceding the gap.
'''
for gap in gaps[::-1]:
if gap[1] - gap[0] > self._max_backfill_block_bodies_at_once:
gap = (BlockNumber(gap[1] - self._max_backfill_block_bodies_at_once), gap[1])
# We want to be sure the header preceding the block gap is in DB
gap_with_prev_block = (BlockNumber(gap[0] - 1), gap[1])
for header_gap in header_gaps[::-1]:
if not self._have_empty_intersection(gap_with_prev_block, header_gap):
break
else:
return gap
else:
raise NoActionableGap
def _have_empty_intersection(self, block_gap: BlockRange, header_gap: BlockRange) -> bool:
return block_gap[0] > header_gap[1] or block_gap[1] < header_gap[0]
@property
def is_paused(self) -> bool:
"""
Return ``True`` if the sync is currently paused, otherwise ``False``.
"""
return self._pauser.is_paused
def pause(self) -> None:
"""
Pause the sync. Pause and resume are wasteful actions and should not happen too frequently.
"""
self._pauser.pause()
if self._body_syncer:
# Pausing the syncer is valid at any point that the service is running but that might
# hit a point where we are still resolving the gaps and have no body_syncer set up yet.
self._body_syncer.get_manager().cancel()
self.logger.debug2("BodyChainGapSyncer paused")
def resume(self) -> None:
"""
Resume the sync.
"""
self._pauser.resume()
self.logger.debug2("BodyChainGapSyncer resumed")
async def run(self) -> None:
"""
Run the sync indefinitely until it is cancelled externally.
"""
while True:
if self._pauser.is_paused:
await self._pauser.await_resume()
try:
await self._setup_for_next_gap()
except ValidationError:
self.logger.debug(
"There are no gaps in the chain of blocks at this time. Sleeping for %ss",
self._idle_time
)
await asyncio.sleep(self._idle_time)
else:
await self.manager.run_service(self._body_syncer)