def listen_steemd(trail_blocks=2):
steemd = get_adapter()
curr_block = db_last_block()
last_hash = False
while True:
curr_block = curr_block + 1
# if trailing too close, take a pause
while trail_blocks > 0:
if curr_block <= steemd.head_block() - trail_blocks:
break
time.sleep(0.5)
# get the target block; if DNE, pause and retry
block = steemd.get_block(curr_block)
while not block:
time.sleep(0.5)
block = steemd.get_block(curr_block)
num = int(block['block_id'][:8], base=16)
print("[LIVE] Got block {} at {} with {} txs -- ".format(
num, block['timestamp'], len(block['transactions'])),
end='')
# ensure the block we received links to our last
if last_hash and last_hash != block['previous']:
# this condition is very rare unless trail_blocks is 0 and fork is
# encountered; to handle gracefully, implement a pop_block method
raise Exception("Unlinkable block: have {}, got {} -> {})".format(
last_hash, block['previous'], block['block_id']))
last_hash = block['block_id']
start_time = time.perf_counter()
query("START TRANSACTION")
dirty = process_block(block)
update_posts_batch(Posts.urls_to_tuples(dirty), steemd,
block['timestamp'])
paidout = select_paidout_posts(block['timestamp'])
update_posts_batch(paidout, steemd, block['timestamp'])
Accounts.cache_dirty()
Accounts.cache_dirty_follows()
print("{} edits, {} payouts".format(len(dirty), len(paidout)))
query("COMMIT")
secs = time.perf_counter() - start_time
if secs > 1:
print("WARNING: block {} process took {}s".format(num, secs))
# approx once per hour, update accounts
if num % 1200 == 0:
print("Performing account maintenance...")
Accounts.cache_old()
Accounts.update_ranks()
def sync_from_steemd(is_initial_sync):
steemd = get_adapter()
dirty = set()
lbound = db_last_block() + 1
ubound = steemd.last_irreversible_block_num()
print("[SYNC] {} blocks to batch sync".format(ubound - lbound + 1))
print("[SYNC] start sync from block %d" % lbound)
while lbound < ubound:
to = min(lbound + 1000, ubound)
lap_0 = time.perf_counter()
blocks = steemd.get_blocks_range(lbound, to)
lap_1 = time.perf_counter()
dirty |= process_blocks(blocks, is_initial_sync)
lap_2 = time.perf_counter()
rate = (to - lbound) / (lap_2 - lap_0)
rps = int((to - lbound) / (lap_1 - lap_0))
wps = int((to - lbound) / (lap_2 - lap_1))
print(
"[SYNC] Got block {} ({}/s, {}rps {}wps) -- {}m remaining".format(
to - 1, round(rate, 1), rps, wps,
round((ubound - to) / rate / 60, 2)))
lbound = to
# batch update post cache after catching up to head block
if not is_initial_sync:
print("[PREP] Update {} edited posts".format(len(dirty)))
update_posts_batch(Posts.urls_to_tuples(dirty), steemd)
date = steemd.head_time()
paidout = select_paidout_posts(date)
print("[PREP] Process {} payouts since {}".format(len(paidout), date))
update_posts_batch(paidout, steemd, date)
