def run(self): """Initialize state; setup/recovery checks; sync and runloop.""" # ensure db schema up to date, check app status DbState.initialize() # prefetch id->name and id->rank memory maps Accounts.load_ids() Accounts.fetch_ranks() Community.recalc_pending_payouts() if DbState.is_initial_sync(): # resume initial sync self.initial() DbState.finish_initial_sync() else: # recover from fork Blocks.verify_head(self._steem) # perform cleanup if process did not exit cleanly CachedPost.recover_missing_posts(self._steem) #audit_cache_missing(self._db, self._steem) #audit_cache_deleted(self._db) self._update_chain_state() if self._conf.get('test_max_block'): # debug mode: partial sync return self.from_steemd() if self._conf.get('test_disable_sync'): # debug mode: no sync, just stream return self.listen() while True: # sync up to irreversible block self.from_steemd() # take care of payout backlog CachedPost.dirty_paidouts(Blocks.head_date()) CachedPost.flush(self._steem, trx=True) try: # listen for new blocks self.listen() except MicroForkException as e: # attempt to recover by restarting stream log.error("NOTIFYALERT microfork: %s", repr(e))
def listen(self): """Live (block following) mode.""" trail_blocks = self._conf.get('trail_blocks') assert trail_blocks >= 0 assert trail_blocks <= 100 # debug: no max gap if disable_sync in effect max_gap = None if self._conf.get('test_disable_sync') else 100 steemd = self._steem hive_head = Blocks.head_num() for block in steemd.stream_blocks(hive_head + 1, trail_blocks, max_gap): start_time = perf() self._db.query("START TRANSACTION") num = Blocks.process(block) follows = Follow.flush(trx=False) accts = Accounts.flush(steemd, trx=False, spread=8) CachedPost.dirty_paidouts(block['timestamp']) cnt = CachedPost.flush(steemd, trx=False) self._db.query("COMMIT") ms = (perf() - start_time) * 1000 log.info( "[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits," "% 3d payouts,% 3d votes,% 3d counts,% 3d accts,% 3d follows" " --% 5dms%s", num, block['timestamp'], len(block['transactions']), cnt['insert'], cnt['update'], cnt['payout'], cnt['upvote'], cnt['recount'], accts, follows, ms, ' SLOW' if ms > 1000 else '') if num % 1200 == 0: #1hr log.warning("head block %d @ %s", num, block['timestamp']) log.info("[LIVE] hourly stats") Accounts.fetch_ranks() #Community.recalc_pending_payouts() if num % 200 == 0: #10min Community.recalc_pending_payouts() if num % 100 == 0: #5min log.info("[LIVE] 5-min stats") Accounts.dirty_oldest(500) if num % 20 == 0: #1min self._update_chain_state()