def run_sync():
print("[HIVE] Welcome to hivemind")
# make sure db schema is up to date, perform checks
DbState.initialize()
# prefetch id->name memory map
Accounts.load_ids()
if DbState.is_initial_sync():
# resume initial sync
Sync.initial()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head()
# perform cleanup in case process did not exit cleanly
CachedPost.recover_missing_posts()
while True:
# sync up to irreversible block
Sync.from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(trx=True)
# start listening
Sync.listen()
def run(cls):
"""Initialize state; setup/recovery checks; sync and runloop."""
# ensure db schema up to date, check app status
DbState.initialize()
# prefetch id->name memory map
Accounts.load_ids()
if DbState.is_initial_sync():
# resume initial sync
cls.initial()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head()
# perform cleanup if process did not exit cleanly
CachedPost.recover_missing_posts()
# debug mode: no sync, just stream
if Conf.get('disable_sync'):
return cls.listen()
while True:
# sync up to irreversible block
cls.from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(trx=True)
# listen for new blocks
cls.listen()
def audit_cache_missing(db, steem):
"""Scan all posts to check for missing cache entries."""
last_id = _last_cached_post_id(db)
step = 1000000
steps = int(last_id / step) + 1
log.info("last post id: %d, batches: %d", last_id, steps)
sql = """
SELECT hp.id, hp.author, hp.permlink
FROM hive_posts hp
LEFT JOIN hive_posts_cache hpc
ON hp.id = hpc.post_id
WHERE hp.is_deleted = False
AND hp.id BETWEEN :lbound AND :ubound
AND hpc.post_id IS NULL"""
for idx in range(steps):
lbound = (idx * step) + 1
ubound = (idx + 1) * step
missing = db.query_all(sql, lbound=lbound, ubound=ubound)
log.info("%d <= id <= %d: %d missing", lbound, ubound, len(missing))
for row in missing:
CachedPost.insert(row['author'], row['permlink'], row['id'])
CachedPost.flush(steem, trx=True)
def sync_from_steemd():
is_initial_sync = DbState.is_initial_sync()
steemd = get_adapter()
lbound = Blocks.head_num() + 1
ubound = steemd.last_irreversible()
if ubound <= lbound:
return
_abort = False
try:
print("[SYNC] start block %d, +%d to sync" % (lbound, ubound-lbound+1))
timer = Timer(ubound - lbound, entity='block', laps=['rps', 'wps'])
while lbound < ubound:
to = min(lbound + 1000, ubound)
timer.batch_start()
blocks = steemd.get_blocks_range(lbound, to)
timer.batch_lap()
Blocks.process_multi(blocks, is_initial_sync)
timer.batch_finish(len(blocks))
date = blocks[-1]['timestamp']
print(timer.batch_status("[SYNC] Got block %d @ %s" % (to-1, date)))
lbound = to
except KeyboardInterrupt:
traceback.print_exc()
print("\n\n[SYNC] Aborted.. cleaning up..")
_abort = True
if not is_initial_sync:
# Follows flushing may need to be moved closer to core (i.e. moved
# into main block transactions). Important to keep in sync since
# we need to prevent expensive recounts. This will fail if we aborted
# in the middle of a transaction, meaning data loss. Better than
# forcing it, however, since in-memory cache will be out of sync
# with db state.
Follow.flush(trx=True)
# This flush is low importance; accounts are swept regularly.
if not _abort:
Accounts.flush(trx=True)
# If this flush fails, all that could potentially be lost here is
# edits and pre-payout votes. If the post has not been paid out yet,
# then the worst case is it will be synced upon payout. If the post
# is already paid out, worst case is to lose an edit.
CachedPost.flush(trx=True)
if _abort:
print("[SYNC] Aborted")
exit()
def run(self):
"""Initialize state; setup/recovery checks; sync and runloop."""
# ensure db schema up to date, check app status
DbState.initialize()
# prefetch id->name and id->rank memory maps
Accounts.load_ids()
Accounts.fetch_ranks()
Community.recalc_pending_payouts()
if DbState.is_initial_sync():
# resume initial sync
self.initial()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head(self._steem)
# perform cleanup if process did not exit cleanly
CachedPost.recover_missing_posts(self._steem)
#audit_cache_missing(self._db, self._steem)
#audit_cache_deleted(self._db)
self._update_chain_state()
if self._conf.get('test_max_block'):
# debug mode: partial sync
return self.from_steemd()
if self._conf.get('test_disable_sync'):
# debug mode: no sync, just stream
return self.listen()
while True:
# sync up to irreversible block
self.from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(self._steem, trx=True)
try:
# listen for new blocks
self.listen()
except MicroForkException as e:
# attempt to recover by restarting stream
log.error("NOTIFYALERT microfork: %s", repr(e))
def from_steemd(cls, is_initial_sync=False, chunk_size=1000):
"""Fast sync strategy: read/process blocks in batches."""
steemd = SteemClient.instance()
lbound = Blocks.head_num() + 1
ubound = steemd.last_irreversible()
count = ubound - lbound
if count < 1:
return
_abort = False
try:
print("[SYNC] start block %d, +%d to sync" % (lbound, count))
timer = Timer(count, entity='block', laps=['rps', 'wps'])
while lbound < ubound:
timer.batch_start()
# fetch blocks
to = min(lbound + chunk_size, ubound)
blocks = steemd.get_blocks_range(lbound, to)
lbound = to
timer.batch_lap()
# process blocks
Blocks.process_multi(blocks, is_initial_sync)
timer.batch_finish(len(blocks))
date = blocks[-1]['timestamp']
print(
timer.batch_status("[SYNC] Got block %d @ %s" %
(to - 1, date)))
except KeyboardInterrupt:
traceback.print_exc()
print("\n\n[SYNC] Aborted.. cleaning up..")
_abort = True
if not is_initial_sync:
# This flush is low importance; accounts are swept regularly.
if not _abort:
Accounts.flush(trx=True)
# If this flush fails, all that could potentially be lost here is
# edits and pre-payout votes. If the post has not been paid out yet,
# then the worst case is it will be synced upon payout. If the post
# is already paid out, worst case is to lose an edit.
CachedPost.flush(trx=True)
if _abort:
print("[SYNC] Aborted")
exit()
def listen(cls):
trail_blocks = Conf.get('trail_blocks')
assert trail_blocks >= 0
assert trail_blocks < 25
steemd = SteemClient.instance()
hive_head = Blocks.head_num()
for block in steemd.stream_blocks(hive_head + 1, trail_blocks, max_gap=40):
start_time = time.perf_counter()
query("START TRANSACTION")
num = Blocks.process(block)
follows = Follow.flush(trx=False)
accts = Accounts.flush(trx=False, period=8)
CachedPost.dirty_paidouts(block['timestamp'])
cnt = CachedPost.flush(trx=False)
query("COMMIT")
ms = (time.perf_counter() - start_time) * 1000
print("[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits,"
"% 3d payouts,% 3d votes,% 3d accounts,% 3d follows --% 5dms%s"
% (num, block['timestamp'], len(block['transactions']),
cnt['insert'], cnt['update'], cnt['payout'], cnt['upvote'],
accts, follows, int(ms), ' SLOW' if ms > 1000 else ''))
# once per hour, update accounts
if num % 1200 == 0:
Accounts.dirty_oldest(10000)
Accounts.flush(trx=True)
#Accounts.update_ranks()
# once a minute, update chain props
if num % 20 == 0:
cls._update_chain_state(steemd)
def from_dpayd(self, is_initial_sync=False, chunk_size=1000):
"""Fast sync strategy: read/process blocks in batches."""
# pylint: disable=no-self-use
dpayd = self._dpay
lbound = Blocks.head_num() + 1
ubound = self._conf.get('test_max_block') or dpayd.last_irreversible()
count = ubound - lbound
if count < 1:
return
log.info("[SYNC] start block %d, +%d to sync", lbound, count)
timer = Timer(count, entity='block', laps=['rps', 'wps'])
while lbound < ubound:
timer.batch_start()
# fetch blocks
to = min(lbound + chunk_size, ubound)
blocks = dpayd.get_blocks_range(lbound, to)
lbound = to
timer.batch_lap()
# process blocks
Blocks.process_multi(blocks, is_initial_sync)
timer.batch_finish(len(blocks))
_prefix = ("[SYNC] Got block %d @ %s" %
(to - 1, blocks[-1]['timestamp']))
log.info(timer.batch_status(_prefix))
if not is_initial_sync:
# This flush is low importance; accounts are swept regularly.
Accounts.flush(dpayd, trx=True)
# If this flush fails, all that could potentially be lost here is
# edits and pre-payout votes. If the post has not been paid out yet,
# then the worst case is it will be synced upon payout. If the post
# is already paid out, worst case is to lose an edit.
CachedPost.flush(dpayd, trx=True)
def run():
print("[HIVE] Welcome to hivemind")
# make sure db schema is up to date, perform checks
DbState.initialize()
# prefetch id->name memory map
Accounts.load_ids()
if DbState.is_initial_sync():
print("[INIT] *** Initial fast sync ***")
sync_from_checkpoints()
sync_from_steemd()
print("[INIT] *** Initial cache build ***")
# todo: disable indexes during this process
cache_missing_posts()
FeedCache.rebuild()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head()
# perform cleanup in case process did not exit cleanly
cache_missing_posts()
while True:
# sync up to irreversible block
sync_from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(trx=True)
# start listening
listen_steemd()
def audit_cache_undelete(db, steem):
"""Scan all posts to check for posts erroneously deleted."""
last_id = _last_post_id(db)
step = 1000000
steps = int(last_id / step) + 1
log.info("last post id: %d, batches: %d", last_id, steps)
sql = """
SELECT id, author, permlink
FROM hive_posts
WHERE is_deleted = True
AND id BETWEEN :lbound AND :ubound
"""
for idx in range(steps):
lbound = (idx * step) + 1
ubound = (idx + 1) * step
rows = db.query_all(sql, lbound=lbound, ubound=ubound)
log.info("%d <= id <= %d: %d to check", lbound, ubound, len(rows))
if not rows:
continue
post_args = [(row['author'], row['permlink']) for row in rows]
posts = steem.get_content_batch(post_args)
recovered = 0
for row, post in zip(rows, posts):
if post['author']:
recovered += 1
Posts.undelete(post, post['created'], row['id'])
log.info("%d <= id <= %d: %d recovered", lbound, ubound, recovered)
if recovered:
CachedPost.flush(steem, trx=True)
def listen(self):
"""Live (block following) mode."""
trail_blocks = self._conf.get('trail_blocks')
assert trail_blocks >= 0
assert trail_blocks <= 100
# debug: no max gap if disable_sync in effect
max_gap = None if self._conf.get('test_disable_sync') else 100
steemd = self._steem
hive_head = Blocks.head_num()
for block in steemd.stream_blocks(hive_head + 1, trail_blocks,
max_gap):
start_time = perf()
self._db.query("START TRANSACTION")
num = Blocks.process(block)
follows = Follow.flush(trx=False)
accts = Accounts.flush(steemd, trx=False, spread=8)
CachedPost.dirty_paidouts(block['timestamp'])
cnt = CachedPost.flush(steemd, trx=False)
self._db.query("COMMIT")
ms = (perf() - start_time) * 1000
log.info(
"[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits,"
"% 3d payouts,% 3d votes,% 3d counts,% 3d accts,% 3d follows"
" --% 5dms%s", num, block['timestamp'],
len(block['transactions']), cnt['insert'], cnt['update'],
cnt['payout'], cnt['upvote'], cnt['recount'], accts, follows,
ms, ' SLOW' if ms > 1000 else '')
if num % 1200 == 0: #1hr
log.warning("head block %d @ %s", num, block['timestamp'])
log.info("[LIVE] hourly stats")
Accounts.fetch_ranks()
#Community.recalc_pending_payouts()
if num % 200 == 0: #10min
Community.recalc_pending_payouts()
if num % 100 == 0: #5min
log.info("[LIVE] 5-min stats")
Accounts.dirty_oldest(500)
if num % 20 == 0: #1min
self._update_chain_state()
def listen(cls):
"""Live (block following) mode."""
trail_blocks = Conf.get('trail_blocks')
assert trail_blocks >= 0
assert trail_blocks <= 100
# debug: no max gap if disable_sync in effect
max_gap = None if Conf.get('disable_sync') else 100
steemd = SteemClient.instance()
hive_head = Blocks.head_num()
for block in steemd.stream_blocks(hive_head + 1, trail_blocks,
max_gap):
start_time = perf()
query("START TRANSACTION")
num = Blocks.process(block)
follows = Follow.flush(trx=False)
accts = Accounts.flush(trx=False, spread=8)
CachedPost.dirty_paidouts(block['timestamp'])
cnt = CachedPost.flush(trx=False)
query("COMMIT")
ms = (perf() - start_time) * 1000
log.info(
"[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits,"
"% 3d payouts,% 3d votes,% 3d accts,% 3d follows --% 5dms%s",
num, block['timestamp'], len(block['transactions']),
cnt['insert'], cnt['update'], cnt['payout'], cnt['upvote'],
accts, follows, int(ms), ' SLOW' if ms > 1000 else '')
# once per hour, update accounts
if num % 1200 == 0:
Accounts.dirty_oldest(10000)
Accounts.flush(trx=True)
#Accounts.update_ranks()
# once a minute, update chain props
if num % 20 == 0:
cls._update_chain_state(steemd)
def cache_missing_posts():
gap = CachedPost.dirty_missing()
print("[INIT] {} missing post cache entries".format(gap))
while CachedPost.flush(trx=True)['insert']:
CachedPost.dirty_missing()
def listen_steemd(trail_blocks=0, max_gap=50):
assert trail_blocks >= 0
assert trail_blocks < 25
# db state
db_last = Blocks.last()
last_block = db_last['num']
last_hash = db_last['hash']
# chain state
steemd = get_adapter()
head_block = steemd.head_block()
next_expected = time.time()
# loop state
tries = 0
queue = []
# TODO: detect missed blocks by looking at block timestamps.
# this would be an even more efficient way to track slots.
while True:
assert not last_block > head_block
# fast fwd head block if slots missed
curr_time = time.time()
while curr_time >= next_expected:
head_block += 1
next_expected += 3
# if gap too large, abort. if caught up, wait.
gap = head_block - last_block
if gap > max_gap:
print("[LIVE] gap too large: %d -- abort listen mode" % gap)
return
elif gap > 0:
print("[LIVE] %d blocks behind..." % gap)
elif gap == 0:
time.sleep(next_expected - curr_time)
head_block += 1
next_expected += 3
# get the target block; if DNE, pause and retry
block_num = last_block + 1
block = steemd.get_block(block_num)
if not block:
tries += 1
print("[LIVE] block %d unavailable (try %d). delay 1s. head: %d/%d."
% (block_num, tries, head_block, steemd.head_block()))
#assert tries < 12, "could not fetch block %s" % block_num
assert tries < 240, "could not fetch block %s" % block_num #74
time.sleep(1) # pause for 1s; and,
next_expected += 1 # delay schedule 1s
continue
last_block = block_num
tries = 0
# ensure this block links to our last; otherwise, blow up. see #59
if last_hash != block['previous']:
if queue:
print("[FORK] Fork encountered. Emptying queue to retry!")
return
raise Exception("Unlinkable block: have %s, got %s -> %s)"
% (last_hash, block['previous'], block['block_id']))
last_hash = block['block_id']
# buffer until queue full
queue.append(block)
if len(queue) <= trail_blocks:
continue
# buffer primed; process head of queue
# ------------------------------------
block = queue.pop(0)
start_time = time.perf_counter()
query("START TRANSACTION")
num = Blocks.process(block)
follows = Follow.flush(trx=False)
accts = Accounts.flush(trx=False, period=8)
CachedPost.dirty_paidouts(block['timestamp'])
cnt = CachedPost.flush(trx=False)
query("COMMIT")
ms = (time.perf_counter() - start_time) * 1000
print("[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits,"
"% 3d payouts,% 3d votes,% 3d accounts,% 3d follows --% 5dms%s"
% (num, block['timestamp'], len(block['transactions']),
cnt['insert'], cnt['update'], cnt['payout'], cnt['upvote'],
accts, follows, int(ms), ' SLOW' if ms > 1000 else ''))
# once per hour, update accounts
if num % 1200 == 0:
Accounts.dirty_oldest(10000)
Accounts.flush(trx=True)
#Accounts.update_ranks()
# once a minute, update chain props
if num % 20 == 0:
update_chain_state()
