def run(self):
"""Initialize state; setup/recovery checks; sync and runloop."""
# ensure db schema up to date, check app status
DbState.initialize()
# prefetch id->name and id->rank memory maps
Accounts.load_ids()
Accounts.fetch_ranks()
Community.recalc_pending_payouts()
if DbState.is_initial_sync():
# resume initial sync
self.initial()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head(self._steem)
# perform cleanup if process did not exit cleanly
CachedPost.recover_missing_posts(self._steem)
#audit_cache_missing(self._db, self._steem)
#audit_cache_deleted(self._db)
self._update_chain_state()
if self._conf.get('test_max_block'):
# debug mode: partial sync
return self.from_steemd()
if self._conf.get('test_disable_sync'):
# debug mode: no sync, just stream
return self.listen()
while True:
# sync up to irreversible block
self.from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(self._steem, trx=True)
try:
# listen for new blocks
self.listen()
except MicroForkException as e:
# attempt to recover by restarting stream
log.error("NOTIFYALERT microfork: %s", repr(e))
def listen(self):
"""Live (block following) mode."""
trail_blocks = self._conf.get('trail_blocks')
assert trail_blocks >= 0
assert trail_blocks <= 100
# debug: no max gap if disable_sync in effect
max_gap = None if self._conf.get('test_disable_sync') else 100
steemd = self._steem
hive_head = Blocks.head_num()
for block in steemd.stream_blocks(hive_head + 1, trail_blocks,
max_gap):
start_time = perf()
self._db.query("START TRANSACTION")
num = Blocks.process(block)
follows = Follow.flush(trx=False)
accts = Accounts.flush(steemd, trx=False, spread=8)
CachedPost.dirty_paidouts(block['timestamp'])
cnt = CachedPost.flush(steemd, trx=False)
self._db.query("COMMIT")
ms = (perf() - start_time) * 1000
log.info(
"[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits,"
"% 3d payouts,% 3d votes,% 3d counts,% 3d accts,% 3d follows"
" --% 5dms%s", num, block['timestamp'],
len(block['transactions']), cnt['insert'], cnt['update'],
cnt['payout'], cnt['upvote'], cnt['recount'], accts, follows,
ms, ' SLOW' if ms > 1000 else '')
if num % 1200 == 0: #1hr
log.warning("head block %d @ %s", num, block['timestamp'])
log.info("[LIVE] hourly stats")
Accounts.fetch_ranks()
#Community.recalc_pending_payouts()
if num % 200 == 0: #10min
Community.recalc_pending_payouts()
if num % 100 == 0: #5min
log.info("[LIVE] 5-min stats")
Accounts.dirty_oldest(500)
if num % 20 == 0: #1min
self._update_chain_state()