class BodyChainGapSyncer(Service): """ A service to sync historical blocks without executing them. This service is meant to be run in tandem with other operations that sync the state. """ _idle_time = BLOCK_BACKFILL_IDLE_TIME _starting_tip: BlockHeaderAPI = None logger = get_logger('trinity.sync.beam.chain.BodyChainGapSyncer') def __init__(self, chain: AsyncChainAPI, db: BaseAsyncChainDB, peer_pool: ETHPeerPool) -> None: self._chain = chain self._db = db self._peer_pool = peer_pool self._pauser = Pauser() self._body_syncer: FastChainBodySyncer = None self._max_backfill_block_bodies_at_once = MAX_BACKFILL_BLOCK_BODIES_AT_ONCE async def _setup_for_next_gap(self) -> None: gap_start, gap_end = self._get_next_gap() fill_start = BlockNumber(max( gap_start, gap_end - self._max_backfill_block_bodies_at_once, )) start_num = BlockNumber(fill_start - 1) _starting_tip = await self._db.coro_get_canonical_block_header_by_number(start_num) if self._pauser.is_paused: # If the syncer was paused while we were busy setting it up throw the current setup # away. A new setup will be performed as soon as `resume()` was called again. raise ValidationError("Syncer was paused by the user") async def _get_launch_header() -> BlockHeaderAPI: return _starting_tip self.logger.debug("Starting to sync missing blocks from #%s to #%s", fill_start, gap_end) self._body_syncer = FastChainBodySyncer( self._chain, self._db, self._peer_pool, DatabaseBlockRangeHeaderSyncer(self._db, (fill_start, gap_end,)), launch_header_fn=_get_launch_header, should_skip_header_fn=body_for_header_exists(self._db, self._chain) ) self._body_syncer.logger = self.logger def _get_next_gap(self) -> BlockRange: gaps, future_tip_block = self._db.get_chain_gaps() header_gaps, future_tip_header = self._db.get_header_chain_gaps() try: actionable_gap = self.get_topmost_actionable_gap(gaps, header_gaps) except NoActionableGap: # We do not have gaps in the chain of blocks but we may still have a gap from the last # block up until the highest consecutive written header. if len(header_gaps) > 0: # The header chain has gaps, find out the lowest missing header lowest_missing_header, _ = header_gaps[0] else: # It doesn't have gaps, so the future_tip_header is the lowest missing header lowest_missing_header = future_tip_header highest_consecutive_header = lowest_missing_header - 1 if highest_consecutive_header >= future_tip_block: # The header before the lowest missing header is the highest consecutive header # that exists in the db and it is higher than the future tip block. That's a gap # we can try to close. return future_tip_block, BlockNumber(highest_consecutive_header) else: raise ValidationError("No gaps in the chain of blocks") else: return actionable_gap def get_topmost_actionable_gap(self, gaps: Tuple[BlockRange, ...], header_gaps: Tuple[BlockRange, ...]) -> BlockRange: ''' Returns the most recent gap of blocks of max size = _max_backfill_block_bodies_at_once for which the headers exist in DB, along with the header preceding the gap. ''' for gap in gaps[::-1]: if gap[1] - gap[0] > self._max_backfill_block_bodies_at_once: gap = (BlockNumber(gap[1] - self._max_backfill_block_bodies_at_once), gap[1]) # We want to be sure the header preceding the block gap is in DB gap_with_prev_block = (BlockNumber(gap[0] - 1), gap[1]) for header_gap in header_gaps[::-1]: if not self._have_empty_intersection(gap_with_prev_block, header_gap): break else: return gap else: raise NoActionableGap def _have_empty_intersection(self, block_gap: BlockRange, header_gap: BlockRange) -> bool: return block_gap[0] > header_gap[1] or block_gap[1] < header_gap[0] @property def is_paused(self) -> bool: """ Return ``True`` if the sync is currently paused, otherwise ``False``. """ return self._pauser.is_paused def pause(self) -> None: """ Pause the sync. Pause and resume are wasteful actions and should not happen too frequently. """ self._pauser.pause() if self._body_syncer: # Pausing the syncer is valid at any point that the service is running but that might # hit a point where we are still resolving the gaps and have no body_syncer set up yet. self._body_syncer.get_manager().cancel() self.logger.debug2("BodyChainGapSyncer paused") def resume(self) -> None: """ Resume the sync. """ self._pauser.resume() self.logger.debug2("BodyChainGapSyncer resumed") async def run(self) -> None: """ Run the sync indefinitely until it is cancelled externally. """ while True: if self._pauser.is_paused: await self._pauser.await_resume() try: await self._setup_for_next_gap() except ValidationError: self.logger.debug( "There are no gaps in the chain of blocks at this time. Sleeping for %ss", self._idle_time ) await asyncio.sleep(self._idle_time) else: await self.manager.run_service(self._body_syncer)
class BodyChainGapSyncer(Service): """ A service to sync historical blocks without executing them. This service is meant to be run in tandem with other operations that sync the state. """ _starting_tip: BlockHeader = None logger = get_logger('trinity.sync.beam.chain.BodyChainGapSyncer') def __init__(self, chain: AsyncChainAPI, db: BaseAsyncChainDB, peer_pool: ETHPeerPool) -> None: self._chain = chain self._db = db self._peer_pool = peer_pool self._paused = False self._resumed = asyncio.Event() async def _setup_for_next_gap(self) -> None: gap_start, gap_end = self._get_next_gap() start_num = BlockNumber(gap_start - 1) _starting_tip = await self._db.coro_get_canonical_block_header_by_number(start_num) async def _get_launch_header() -> BlockHeaderAPI: return _starting_tip self.logger.debug("Starting to sync missing blocks from #%s to #%s", gap_start, gap_end) self._body_syncer = FastChainBodySyncer( self._chain, self._db, self._peer_pool, DatabaseBlockRangeHeaderSyncer(self._db, (gap_start, gap_end,)), launch_header_fn=_get_launch_header, should_skip_header_fn=body_for_header_exists(self._db, self._chain) ) def _get_next_gap(self) -> BlockRange: gaps, future_tip_block = self._db.get_chain_gaps() if len(gaps) == 0: # We do not have gaps in the chain of blocks but we may still have a gap from the last # block up until the highest consecutive written header. header_gaps, future_tip_header = self._db.get_header_chain_gaps() if len(header_gaps) > 0: # The header chain has gaps, find out the lowest missing header lowest_missing_header, _ = header_gaps[0] else: # It doesn't have gaps, so the future_tip_header is the lowest missing header lowest_missing_header = future_tip_header highest_consecutive_header = lowest_missing_header - 1 if highest_consecutive_header >= future_tip_block: # The header before the lowest missing header is the highest consecutive header # that exists in the db and it is higher than the future tip block. That's a gap # we can try to close. return future_tip_block, BlockNumber(highest_consecutive_header) else: raise ValidationError("No gaps in the chain of blocks") else: return gaps[0] @property def is_paused(self) -> bool: """ Return ``True`` if the sync is currently paused, otherwise ``False``. """ return self._paused def pause(self) -> None: """ Pause the sync. Pause and resume are wasteful actions and should not happen too frequently. """ if self._paused: raise RuntimeError("Invalid action. Can not pause service that is already paused.") self._paused = True self._body_syncer.get_manager().cancel() self.logger.debug2("BodyChainGapSyncer paused") def resume(self) -> None: """ Resume the sync. """ if not self._paused: raise RuntimeError("Invalid action. Can not resume service that isn't paused.") self._paused = False self.logger.debug2("BodyChainGapSyncer resumed") self._resumed.set() async def run(self) -> None: """ Run the sync indefinitely until it is cancelled externally. """ while True: if self._paused: await self._resumed.wait() self._resumed.clear() try: await self._setup_for_next_gap() except ValidationError: self.logger.debug( "There are no gaps in the chain of blocks at this time. Sleeping for %ss", BLOCK_BACKFILL_IDLE_TIME ) await asyncio.sleep(BLOCK_BACKFILL_IDLE_TIME) else: await self.manager.run_service(self._body_syncer)