class BodyChainGapSyncer(Service):
"""
A service to sync historical blocks without executing them. This service is meant to be run
in tandem with other operations that sync the state.
"""
_idle_time = BLOCK_BACKFILL_IDLE_TIME
_starting_tip: BlockHeaderAPI = None
logger = get_logger('trinity.sync.beam.chain.BodyChainGapSyncer')
def __init__(self,
chain: AsyncChainAPI,
db: BaseAsyncChainDB,
peer_pool: ETHPeerPool) -> None:
self._chain = chain
self._db = db
self._peer_pool = peer_pool
self._pauser = Pauser()
self._body_syncer: FastChainBodySyncer = None
self._max_backfill_block_bodies_at_once = MAX_BACKFILL_BLOCK_BODIES_AT_ONCE
async def _setup_for_next_gap(self) -> None:
gap_start, gap_end = self._get_next_gap()
fill_start = BlockNumber(max(
gap_start,
gap_end - self._max_backfill_block_bodies_at_once,
))
start_num = BlockNumber(fill_start - 1)
_starting_tip = await self._db.coro_get_canonical_block_header_by_number(start_num)
if self._pauser.is_paused:
# If the syncer was paused while we were busy setting it up throw the current setup
# away. A new setup will be performed as soon as `resume()` was called again.
raise ValidationError("Syncer was paused by the user")
async def _get_launch_header() -> BlockHeaderAPI:
return _starting_tip
self.logger.debug("Starting to sync missing blocks from #%s to #%s", fill_start, gap_end)
self._body_syncer = FastChainBodySyncer(
self._chain,
self._db,
self._peer_pool,
DatabaseBlockRangeHeaderSyncer(self._db, (fill_start, gap_end,)),
launch_header_fn=_get_launch_header,
should_skip_header_fn=body_for_header_exists(self._db, self._chain)
)
self._body_syncer.logger = self.logger
def _get_next_gap(self) -> BlockRange:
gaps, future_tip_block = self._db.get_chain_gaps()
header_gaps, future_tip_header = self._db.get_header_chain_gaps()
try:
actionable_gap = self.get_topmost_actionable_gap(gaps, header_gaps)
except NoActionableGap:
# We do not have gaps in the chain of blocks but we may still have a gap from the last
# block up until the highest consecutive written header.
if len(header_gaps) > 0:
# The header chain has gaps, find out the lowest missing header
lowest_missing_header, _ = header_gaps[0]
else:
# It doesn't have gaps, so the future_tip_header is the lowest missing header
lowest_missing_header = future_tip_header
highest_consecutive_header = lowest_missing_header - 1
if highest_consecutive_header >= future_tip_block:
# The header before the lowest missing header is the highest consecutive header
# that exists in the db and it is higher than the future tip block. That's a gap
# we can try to close.
return future_tip_block, BlockNumber(highest_consecutive_header)
else:
raise ValidationError("No gaps in the chain of blocks")
else:
return actionable_gap
def get_topmost_actionable_gap(self,
gaps: Tuple[BlockRange, ...],
header_gaps: Tuple[BlockRange, ...]) -> BlockRange:
'''
Returns the most recent gap of blocks of max size = _max_backfill_block_bodies_at_once
for which the headers exist in DB, along with the header preceding the gap.
'''
for gap in gaps[::-1]:
if gap[1] - gap[0] > self._max_backfill_block_bodies_at_once:
gap = (BlockNumber(gap[1] - self._max_backfill_block_bodies_at_once), gap[1])
# We want to be sure the header preceding the block gap is in DB
gap_with_prev_block = (BlockNumber(gap[0] - 1), gap[1])
for header_gap in header_gaps[::-1]:
if not self._have_empty_intersection(gap_with_prev_block, header_gap):
break
else:
return gap
else:
raise NoActionableGap
def _have_empty_intersection(self, block_gap: BlockRange, header_gap: BlockRange) -> bool:
return block_gap[0] > header_gap[1] or block_gap[1] < header_gap[0]
@property
def is_paused(self) -> bool:
"""
Return ``True`` if the sync is currently paused, otherwise ``False``.
"""
return self._pauser.is_paused
def pause(self) -> None:
"""
Pause the sync. Pause and resume are wasteful actions and should not happen too frequently.
"""
self._pauser.pause()
if self._body_syncer:
# Pausing the syncer is valid at any point that the service is running but that might
# hit a point where we are still resolving the gaps and have no body_syncer set up yet.
self._body_syncer.get_manager().cancel()
self.logger.debug2("BodyChainGapSyncer paused")
def resume(self) -> None:
"""
Resume the sync.
"""
self._pauser.resume()
self.logger.debug2("BodyChainGapSyncer resumed")
async def run(self) -> None:
"""
Run the sync indefinitely until it is cancelled externally.
"""
while True:
if self._pauser.is_paused:
await self._pauser.await_resume()
try:
await self._setup_for_next_gap()
except ValidationError:
self.logger.debug(
"There are no gaps in the chain of blocks at this time. Sleeping for %ss",
self._idle_time
)
await asyncio.sleep(self._idle_time)
else:
await self.manager.run_service(self._body_syncer)
class BodyChainGapSyncer(Service):
"""
A service to sync historical blocks without executing them. This service is meant to be run
in tandem with other operations that sync the state.
"""
_starting_tip: BlockHeader = None
logger = get_logger('trinity.sync.beam.chain.BodyChainGapSyncer')
def __init__(self,
chain: AsyncChainAPI,
db: BaseAsyncChainDB,
peer_pool: ETHPeerPool) -> None:
self._chain = chain
self._db = db
self._peer_pool = peer_pool
self._paused = False
self._resumed = asyncio.Event()
async def _setup_for_next_gap(self) -> None:
gap_start, gap_end = self._get_next_gap()
start_num = BlockNumber(gap_start - 1)
_starting_tip = await self._db.coro_get_canonical_block_header_by_number(start_num)
async def _get_launch_header() -> BlockHeaderAPI:
return _starting_tip
self.logger.debug("Starting to sync missing blocks from #%s to #%s", gap_start, gap_end)
self._body_syncer = FastChainBodySyncer(
self._chain,
self._db,
self._peer_pool,
DatabaseBlockRangeHeaderSyncer(self._db, (gap_start, gap_end,)),
launch_header_fn=_get_launch_header,
should_skip_header_fn=body_for_header_exists(self._db, self._chain)
)
def _get_next_gap(self) -> BlockRange:
gaps, future_tip_block = self._db.get_chain_gaps()
if len(gaps) == 0:
# We do not have gaps in the chain of blocks but we may still have a gap from the last
# block up until the highest consecutive written header.
header_gaps, future_tip_header = self._db.get_header_chain_gaps()
if len(header_gaps) > 0:
# The header chain has gaps, find out the lowest missing header
lowest_missing_header, _ = header_gaps[0]
else:
# It doesn't have gaps, so the future_tip_header is the lowest missing header
lowest_missing_header = future_tip_header
highest_consecutive_header = lowest_missing_header - 1
if highest_consecutive_header >= future_tip_block:
# The header before the lowest missing header is the highest consecutive header
# that exists in the db and it is higher than the future tip block. That's a gap
# we can try to close.
return future_tip_block, BlockNumber(highest_consecutive_header)
else:
raise ValidationError("No gaps in the chain of blocks")
else:
return gaps[0]
@property
def is_paused(self) -> bool:
"""
Return ``True`` if the sync is currently paused, otherwise ``False``.
"""
return self._paused
def pause(self) -> None:
"""
Pause the sync. Pause and resume are wasteful actions and should not happen too frequently.
"""
if self._paused:
raise RuntimeError("Invalid action. Can not pause service that is already paused.")
self._paused = True
self._body_syncer.get_manager().cancel()
self.logger.debug2("BodyChainGapSyncer paused")
def resume(self) -> None:
"""
Resume the sync.
"""
if not self._paused:
raise RuntimeError("Invalid action. Can not resume service that isn't paused.")
self._paused = False
self.logger.debug2("BodyChainGapSyncer resumed")
self._resumed.set()
async def run(self) -> None:
"""
Run the sync indefinitely until it is cancelled externally.
"""
while True:
if self._paused:
await self._resumed.wait()
self._resumed.clear()
try:
await self._setup_for_next_gap()
except ValidationError:
self.logger.debug(
"There are no gaps in the chain of blocks at this time. Sleeping for %ss",
BLOCK_BACKFILL_IDLE_TIME
)
await asyncio.sleep(BLOCK_BACKFILL_IDLE_TIME)
else:
await self.manager.run_service(self._body_syncer)
