def delete(cls, op):
"""Marks a post record as being deleted."""
pid, depth = cls.get_id_and_depth(op['author'], op['permlink'])
DB.query("UPDATE hive_posts SET is_deleted = '1' WHERE id = :id",
id=pid)
if not DbState.is_initial_sync():
CachedPost.delete(pid, op['author'], op['permlink'])
if depth == 0:
FeedCache.delete(pid)
else:
# force parent child recount when child is deleted
prnt = cls._get_parent_by_child_id(pid)
CachedPost.recount(prnt['author'], prnt['permlink'],
prnt['id'])
def insert(cls, op, date):
"""Inserts new post records."""
sql = """INSERT INTO hive_posts (is_valid, parent_id, author, permlink,
category, community, depth, created_at)
VALUES (:is_valid, :parent_id, :author, :permlink,
:category, :community, :depth, :date)"""
sql += ";SELECT currval(pg_get_serial_sequence('hive_posts','id'))"
post = cls._build_post(op, date)
result = DB.query(sql, **post)
post['id'] = int(list(result)[0][0])
cls._set_id(op['author'] + '/' + op['permlink'], post['id'])
if not DbState.is_initial_sync():
CachedPost.insert(op['author'], op['permlink'], post['id'])
cls._insert_feed_cache(post)
def _process(cls, block, is_initial_sync=False):
"""Process a single block. Assumes a trx is open."""
#pylint: disable=too-many-branches
num = cls._push(block)
date = block['timestamp']
account_names = set()
comment_ops = []
json_ops = []
delete_ops = []
for tx_idx, tx in enumerate(block['transactions']):
for operation in tx['operations']:
op_type = operation['type']
op = operation['value']
# account ops
if op_type == 'pow_operation':
account_names.add(op['worker_account'])
elif op_type == 'pow2_operation':
account_names.add(op['work']['value']['input']['worker_account'])
elif op_type == 'account_create_operation':
account_names.add(op['new_account_name'])
elif op_type == 'account_create_with_delegation_operation':
account_names.add(op['new_account_name'])
elif op_type == 'create_claimed_account_operation':
account_names.add(op['new_account_name'])
# post ops
elif op_type == 'comment_operation':
comment_ops.append(op)
elif op_type == 'delete_comment_operation':
delete_ops.append(op)
elif op_type == 'vote_operation':
if not is_initial_sync:
CachedPost.vote(op['author'], op['permlink'])
# misc ops
elif op_type == 'transfer_operation':
Payments.op_transfer(op, tx_idx, num, date)
elif op_type == 'custom_json_operation':
json_ops.append(op)
Accounts.register(account_names, date) # register any new names
Posts.comment_ops(comment_ops, date) # handle inserts, edits
Posts.delete_ops(delete_ops) # handle post deletion
CustomOp.process_ops(json_ops, num, date) # follow/reblog/community ops
return num
def from_steemd(cls, is_initial_sync=False, chunk_size=1000):
"""Fast sync strategy: read/process blocks in batches."""
steemd = SteemClient.instance()
lbound = Blocks.head_num() + 1
ubound = steemd.last_irreversible()
count = ubound - lbound
if count < 1:
return
_abort = False
try:
print("[SYNC] start block %d, +%d to sync" % (lbound, count))
timer = Timer(count, entity='block', laps=['rps', 'wps'])
while lbound < ubound:
timer.batch_start()
# fetch blocks
to = min(lbound + chunk_size, ubound)
blocks = steemd.get_blocks_range(lbound, to)
lbound = to
timer.batch_lap()
# process blocks
Blocks.process_multi(blocks, is_initial_sync)
timer.batch_finish(len(blocks))
date = blocks[-1]['timestamp']
print(timer.batch_status("[SYNC] Got block %d @ %s" % (to-1, date)))
except KeyboardInterrupt:
traceback.print_exc()
print("\n\n[SYNC] Aborted.. cleaning up..")
_abort = True
if not is_initial_sync:
# This flush is low importance; accounts are swept regularly.
if not _abort:
Accounts.flush(trx=True)
# If this flush fails, all that could potentially be lost here is
# edits and pre-payout votes. If the post has not been paid out yet,
# then the worst case is it will be synced upon payout. If the post
# is already paid out, worst case is to lose an edit.
CachedPost.flush(trx=True)
if _abort:
print("[SYNC] Aborted")
exit()
def _process(cls, block, is_initial_sync=False):
num = cls._push(block)
date = block['timestamp']
account_names = set()
comment_ops = []
json_ops = []
delete_ops = []
voted_authors = set()
for tx_idx, tx in enumerate(block['transactions']):
for operation in tx['operations']:
op_type, op = operation
# account ops
if op_type == 'pow':
account_names.add(op['worker_account'])
elif op_type == 'pow2':
account_names.add(op['work'][1]['input']['worker_account'])
elif op_type == 'account_create':
account_names.add(op['new_account_name'])
elif op_type == 'account_create_with_delegation':
account_names.add(op['new_account_name'])
# post ops
elif op_type == 'comment':
comment_ops.append(op)
elif op_type == 'delete_comment':
delete_ops.append(op)
elif op_type == 'vote':
if not is_initial_sync:
CachedPost.vote(op['author'], op['permlink'])
voted_authors.add(
op['author']) # TODO: move to cachedpost
# misc ops
elif op_type == 'transfer':
Payments.op_transfer(op, tx_idx, num, date)
elif op_type == 'custom_json':
json_ops.append(op)
Accounts.register(account_names, date) # register any new names
Accounts.dirty(voted_authors) # update rep of voted authors
Posts.comment_ops(comment_ops, date) # handle inserts, edits
Posts.delete_ops(delete_ops) # handle post deletion
CustomOp.process_ops(json_ops, num,
date) # follow/reblog/community ops
return num
def listen(self):
"""Live (block following) mode."""
trail_blocks = self._conf.get('trail_blocks')
assert trail_blocks >= 0
assert trail_blocks <= 100
# debug: no max gap if disable_sync in effect
max_gap = None if self._conf.get('test_disable_sync') else 100
steemd = self._steem
hive_head = Blocks.head_num()
for block in steemd.stream_blocks(hive_head + 1, trail_blocks,
max_gap):
start_time = perf()
self._db.query("START TRANSACTION")
num = Blocks.process(block)
follows = Follow.flush(trx=False)
accts = Accounts.flush(steemd, trx=False, spread=8)
CachedPost.dirty_paidouts(block['timestamp'])
cnt = CachedPost.flush(steemd, trx=False)
self._db.query("COMMIT")
ms = (perf() - start_time) * 1000
log.info(
"[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits,"
"% 3d payouts,% 3d votes,% 3d counts,% 3d accts,% 3d follows"
" --% 5dms%s", num, block['timestamp'],
len(block['transactions']), cnt['insert'], cnt['update'],
cnt['payout'], cnt['upvote'], cnt['recount'], accts, follows,
ms, ' SLOW' if ms > 1000 else '')
if num % 1200 == 0: #1hr
log.warning("head block %d @ %s", num, block['timestamp'])
log.info("[LIVE] hourly stats")
Accounts.fetch_ranks()
#Community.recalc_pending_payouts()
if num % 200 == 0: #10min
Community.recalc_pending_payouts()
if num % 100 == 0: #5min
log.info("[LIVE] 5-min stats")
Accounts.dirty_oldest(500)
if num % 20 == 0: #1min
self._update_chain_state()
def undelete(cls, op, date, pid):
"""Re-allocates an existing record flagged as deleted."""
sql = """UPDATE hive_posts SET is_valid = :is_valid,
is_muted = :is_muted, is_deleted = '0', is_pinned = '0',
parent_id = :parent_id, category = :category,
community_id = :community_id, depth = :depth
WHERE id = :id"""
post = cls._build_post(op, date, pid)
DB.query(sql, **post)
if not DbState.is_initial_sync():
if post['error']:
author_id = Accounts.get_id(post['author'])
Notify('error', dst_id=author_id, when=date,
post_id=post['id'], payload=post['error']).write()
CachedPost.undelete(pid, post['author'], post['permlink'],
post['category'])
cls._insert_feed_cache(post)
def listen(cls):
"""Live (block following) mode."""
trail_blocks = Conf.get('trail_blocks')
assert trail_blocks >= 0
assert trail_blocks <= 100
# debug: no max gap if disable_sync in effect
max_gap = None if Conf.get('disable_sync') else 100
steemd = SteemClient.instance()
hive_head = Blocks.head_num()
for block in steemd.stream_blocks(hive_head + 1, trail_blocks,
max_gap):
start_time = time.perf_counter()
query("START TRANSACTION")
num = Blocks.process(block)
follows = Follow.flush(trx=False)
accts = Accounts.flush(trx=False, spread=8)
CachedPost.dirty_paidouts(block['timestamp'])
cnt = CachedPost.flush(trx=False)
query("COMMIT")
ms = (time.perf_counter() - start_time) * 1000
print(
"[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits,"
"% 3d payouts,% 3d votes,% 3d accounts,% 3d follows --% 5dms%s"
% (num, block['timestamp'], len(block['transactions']),
cnt['insert'], cnt['update'], cnt['payout'], cnt['upvote'],
accts, follows, int(ms), ' SLOW' if ms > 1000 else ''))
# once per hour, update accounts
if num % 1200 == 0:
Accounts.dirty_oldest(10000)
Accounts.flush(trx=True)
#Accounts.update_ranks()
# once a minute, update chain props
if num % 20 == 0:
cls._update_chain_state(steemd)
def run(cls):
"""Initialize state; setup/recovery checks; sync and runloop."""
# ensure db schema up to date, check app status
DbState.initialize()
# prefetch id->name memory map
Accounts.load_ids()
if DbState.is_initial_sync():
# resume initial sync
cls.initial()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head()
# perform cleanup if process did not exit cleanly
CachedPost.recover_missing_posts()
# debug mode: no sync, just stream
if Conf.get('disable_sync'):
return cls.listen()
while True:
# sync up to irreversible block
cls.from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(trx=True)
# listen for new blocks
cls.listen()
def run_sync():
print("[HIVE] Welcome to hivemind")
# make sure db schema is up to date, perform checks
DbState.initialize()
# prefetch id->name memory map
Accounts.load_ids()
if DbState.is_initial_sync():
# resume initial sync
Sync.initial()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head()
# perform cleanup in case process did not exit cleanly
CachedPost.recover_missing_posts()
while True:
# sync up to irreversible block
Sync.from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(trx=True)
# start listening
Sync.listen()
def from_dpayd(self, is_initial_sync=False, chunk_size=1000):
"""Fast sync strategy: read/process blocks in batches."""
# pylint: disable=no-self-use
dpayd = self._dpay
lbound = Blocks.head_num() + 1
ubound = self._conf.get('test_max_block') or dpayd.last_irreversible()
count = ubound - lbound
if count < 1:
return
log.info("[SYNC] start block %d, +%d to sync", lbound, count)
timer = Timer(count, entity='block', laps=['rps', 'wps'])
while lbound < ubound:
timer.batch_start()
# fetch blocks
to = min(lbound + chunk_size, ubound)
blocks = dpayd.get_blocks_range(lbound, to)
lbound = to
timer.batch_lap()
# process blocks
Blocks.process_multi(blocks, is_initial_sync)
timer.batch_finish(len(blocks))
_prefix = ("[SYNC] Got block %d @ %s" %
(to - 1, blocks[-1]['timestamp']))
log.info(timer.batch_status(_prefix))
if not is_initial_sync:
# This flush is low importance; accounts are swept regularly.
Accounts.flush(dpayd, trx=True)
# If this flush fails, all that could potentially be lost here is
# edits and pre-payout votes. If the post has not been paid out yet,
# then the worst case is it will be synced upon payout. If the post
# is already paid out, worst case is to lose an edit.
CachedPost.flush(dpayd, trx=True)
def op_transfer(cls, op, tx_idx, num, date):
record = cls._validated(op, tx_idx, num, date)
if not record:
return
# add payment record
insert = DB.build_upsert('hive_payments', 'id', record)
DB.query(insert)
# read current amount
sql = "SELECT promoted FROM hive_posts WHERE id = :id"
curr_amount = DB.query_one(sql, id=record['post_id'])
new_amount = curr_amount + record['amount']
# update post record
sql = "UPDATE hive_posts SET promoted = :val WHERE id = :id"
DB.query(sql, val=new_amount, id=record['post_id'])
# notify cached_post of new promoted balance, and trigger update
CachedPost.update_promoted_amount(record['post_id'], new_amount)
author, permlink = cls._split_url(op['memo'])
CachedPost.vote(author, permlink, record['post_id'])
def insert(cls, op, date):
"""Inserts new post records."""
sql = """INSERT INTO hive_posts (is_valid, is_muted, parent_id, author,
permlink, category, community_id, depth, created_at)
VALUES (:is_valid, :is_muted, :parent_id, :author,
:permlink, :category, :community_id, :depth, :date)"""
sql += ";SELECT currval(pg_get_serial_sequence('hive_posts','id'))"
post = cls._build_post(op, date)
result = DB.query(sql, **post)
post['id'] = int(list(result)[0][0])
cls._set_id(op['author']+'/'+op['permlink'], post['id'])
if not DbState.is_initial_sync():
if post['error']:
author_id = Accounts.get_id(post['author'])
Notify('error', dst_id=author_id, when=date,
post_id=post['id'], payload=post['error']).write()
CachedPost.insert(op['author'], op['permlink'], post['id'])
if op['parent_author']: # update parent's child count
CachedPost.recount(op['parent_author'],
op['parent_permlink'], post['parent_id'])
cls._insert_feed_cache(post)
def run():
print("[HIVE] Welcome to hivemind")
# make sure db schema is up to date, perform checks
DbState.initialize()
# prefetch id->name memory map
Accounts.load_ids()
if DbState.is_initial_sync():
print("[INIT] *** Initial fast sync ***")
sync_from_checkpoints()
sync_from_steemd()
print("[INIT] *** Initial cache build ***")
# todo: disable indexes during this process
cache_missing_posts()
FeedCache.rebuild()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head()
# perform cleanup in case process did not exit cleanly
cache_missing_posts()
while True:
# sync up to irreversible block
sync_from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(trx=True)
# start listening
listen_steemd()
def audit_cache_deleted(db):
"""Scan all posts to check for extraneous cache entries."""
last_id = _last_cached_post_id(db)
step = 1000000
steps = int(last_id / step) + 1
log.info("audit_cache_deleted -- last id: %d, batches: %d", last_id, steps)
sql = """
SELECT hp.id, hp.author, hp.permlink
FROM hive_posts hp
JOIN hive_posts_cache hpc
ON hp.id = hpc.post_id
WHERE hp.id BETWEEN :lbound AND :ubound
AND hp.is_deleted = True"""
for idx in range(steps):
lbound = (idx * step) + 1
ubound = (idx + 1) * step
extra = db.query_all(sql, lbound=lbound, ubound=ubound)
log.info("%d <= id <= %d: %d to delete", lbound, ubound, len(extra))
for row in extra:
CachedPost.delete(row['id'], row['author'], row['permlink'])
def op_transfer(cls, op, tx_idx, num, date):
"""Process raw transfer op; apply balance if valid post promote."""
record = cls._validated(op, tx_idx, num, date)
if not record:
return
# add payment record
sql = DB.build_insert('hive_payments', record, pk='id')
DB.query(sql)
# read current amount
sql = "SELECT promoted FROM hive_posts WHERE id = :id"
curr_amount = DB.query_one(sql, id=record['post_id'])
new_amount = curr_amount + record['amount']
# update post record
sql = "UPDATE hive_posts SET promoted = :val WHERE id = :id"
DB.query(sql, val=new_amount, id=record['post_id'])
# notify cached_post of new promoted balance, and trigger update
if not DbState.is_initial_sync():
CachedPost.update_promoted_amount(record['post_id'], new_amount)
author, permlink = cls._split_url(op['memo'])
CachedPost.vote(author, permlink, record['post_id'])
def audit_cache_undelete(db, steem):
"""Scan all posts to check for posts erroneously deleted."""
last_id = _last_post_id(db)
step = 1000000
steps = int(last_id / step) + 1
log.info("last post id: %d, batches: %d", last_id, steps)
sql = """
SELECT id, author, permlink
FROM hive_posts
WHERE is_deleted = True
AND id BETWEEN :lbound AND :ubound
"""
for idx in range(steps):
lbound = (idx * step) + 1
ubound = (idx + 1) * step
rows = db.query_all(sql, lbound=lbound, ubound=ubound)
log.info("%d <= id <= %d: %d to check", lbound, ubound, len(rows))
if not rows:
continue
post_args = [(row['author'], row['permlink']) for row in rows]
posts = steem.get_content_batch(post_args)
recovered = 0
for row, post in zip(rows, posts):
if post['author']:
recovered += 1
Posts.undelete(post, post['created'], row['id'])
log.info("%d <= id <= %d: %d recovered", lbound, ubound, recovered)
if recovered:
CachedPost.flush(steem, trx=True)
def run(self):
"""Initialize state; setup/recovery checks; sync and runloop."""
# ensure db schema up to date, check app status
DbState.initialize()
# prefetch id->name and id->rank memory maps
Accounts.load_ids()
Accounts.fetch_ranks()
Community.recalc_pending_payouts()
if DbState.is_initial_sync():
# resume initial sync
self.initial()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head(self._steem)
# perform cleanup if process did not exit cleanly
CachedPost.recover_missing_posts(self._steem)
#audit_cache_missing(self._db, self._steem)
#audit_cache_deleted(self._db)
self._update_chain_state()
if self._conf.get('test_max_block'):
# debug mode: partial sync
return self.from_steemd()
if self._conf.get('test_disable_sync'):
# debug mode: no sync, just stream
return self.listen()
while True:
# sync up to irreversible block
self.from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(self._steem, trx=True)
try:
# listen for new blocks
self.listen()
except MicroForkException as e:
# attempt to recover by restarting stream
log.error("NOTIFYALERT microfork: %s", repr(e))
def process(self):
"""Applies a validated operation."""
assert self.valid, 'cannot apply invalid op'
from hive.indexer.cached_post import CachedPost
action = self.action
params = dict(
date=self.date,
community=self.community,
community_id=self.community_id,
actor=self.actor,
actor_id=self.actor_id,
account=self.account,
account_id=self.account_id,
post_id=self.post_id,
role_id=self.role_id,
notes=self.notes,
title=self.title,
)
# Community-level commands
if action == 'updateProps':
bind = ', '.join([k + " = :" + k for k in list(self.props.keys())])
DB.query("UPDATE hive_communities SET %s WHERE id = :id" % bind,
id=self.community_id,
**self.props)
self._notify('set_props',
payload=json.dumps(read_key_dict(self.op, 'props')))
elif action == 'subscribe':
DB.query(
"""INSERT INTO hive_subscriptions
(account_id, community_id, created_at)
VALUES (:actor_id, :community_id, :date)""", **params)
DB.query(
"""UPDATE hive_communities
SET subscribers = subscribers + 1
WHERE id = :community_id""", **params)
self._notify('subscribe')
elif action == 'unsubscribe':
DB.query(
"""DELETE FROM hive_subscriptions
WHERE account_id = :actor_id
AND community_id = :community_id""", **params)
DB.query(
"""UPDATE hive_communities
SET subscribers = subscribers - 1
WHERE id = :community_id""", **params)
# Account-level actions
elif action == 'setRole':
DB.query(
"""INSERT INTO hive_roles
(account_id, community_id, role_id, created_at)
VALUES (:account_id, :community_id, :role_id, :date)
ON CONFLICT (account_id, community_id)
DO UPDATE SET role_id = :role_id""", **params)
self._notify('set_role', payload=Role(self.role_id).name)
elif action == 'setUserTitle':
DB.query(
"""INSERT INTO hive_roles
(account_id, community_id, title, created_at)
VALUES (:account_id, :community_id, :title, :date)
ON CONFLICT (account_id, community_id)
DO UPDATE SET title = :title""", **params)
self._notify('set_label', payload=self.title)
# Post-level actions
elif action == 'mutePost':
DB.query(
"""UPDATE hive_posts SET is_muted = '1'
WHERE id = :post_id""", **params)
self._notify('mute_post', payload=self.notes)
if not DbState.is_initial_sync():
CachedPost.update(self.account, self.permlink, self.post_id)
elif action == 'unmutePost':
DB.query(
"""UPDATE hive_posts SET is_muted = '0'
WHERE id = :post_id""", **params)
self._notify('unmute_post', payload=self.notes)
if not DbState.is_initial_sync():
CachedPost.update(self.account, self.permlink, self.post_id)
elif action == 'pinPost':
DB.query(
"""UPDATE hive_posts SET is_pinned = '1'
WHERE id = :post_id""", **params)
self._notify('pin_post', payload=self.notes)
elif action == 'unpinPost':
DB.query(
"""UPDATE hive_posts SET is_pinned = '0'
WHERE id = :post_id""", **params)
self._notify('unpin_post', payload=self.notes)
elif action == 'flagPost':
self._notify('flag_post', payload=self.notes)
return True
def cache_missing_posts():
gap = CachedPost.dirty_missing()
print("[INIT] {} missing post cache entries".format(gap))
while CachedPost.flush(trx=True)['insert']:
CachedPost.dirty_missing()
def listen_steemd(trail_blocks=0, max_gap=50):
assert trail_blocks >= 0
assert trail_blocks < 25
# db state
db_last = Blocks.last()
last_block = db_last['num']
last_hash = db_last['hash']
# chain state
steemd = get_adapter()
head_block = steemd.head_block()
next_expected = time.time()
# loop state
tries = 0
queue = []
# TODO: detect missed blocks by looking at block timestamps.
# this would be an even more efficient way to track slots.
while True:
assert not last_block > head_block
# fast fwd head block if slots missed
curr_time = time.time()
while curr_time >= next_expected:
head_block += 1
next_expected += 3
# if gap too large, abort. if caught up, wait.
gap = head_block - last_block
if gap > max_gap:
print("[LIVE] gap too large: %d -- abort listen mode" % gap)
return
elif gap > 0:
print("[LIVE] %d blocks behind..." % gap)
elif gap == 0:
time.sleep(next_expected - curr_time)
head_block += 1
next_expected += 3
# get the target block; if DNE, pause and retry
block_num = last_block + 1
block = steemd.get_block(block_num)
if not block:
tries += 1
print("[LIVE] block %d unavailable (try %d). delay 1s. head: %d/%d."
% (block_num, tries, head_block, steemd.head_block()))
#assert tries < 12, "could not fetch block %s" % block_num
assert tries < 240, "could not fetch block %s" % block_num #74
time.sleep(1) # pause for 1s; and,
next_expected += 1 # delay schedule 1s
continue
last_block = block_num
tries = 0
# ensure this block links to our last; otherwise, blow up. see #59
if last_hash != block['previous']:
if queue:
print("[FORK] Fork encountered. Emptying queue to retry!")
return
raise Exception("Unlinkable block: have %s, got %s -> %s)"
% (last_hash, block['previous'], block['block_id']))
last_hash = block['block_id']
# buffer until queue full
queue.append(block)
if len(queue) <= trail_blocks:
continue
# buffer primed; process head of queue
# ------------------------------------
block = queue.pop(0)
start_time = time.perf_counter()
query("START TRANSACTION")
num = Blocks.process(block)
follows = Follow.flush(trx=False)
accts = Accounts.flush(trx=False, period=8)
CachedPost.dirty_paidouts(block['timestamp'])
cnt = CachedPost.flush(trx=False)
query("COMMIT")
ms = (time.perf_counter() - start_time) * 1000
print("[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits,"
"% 3d payouts,% 3d votes,% 3d accounts,% 3d follows --% 5dms%s"
% (num, block['timestamp'], len(block['transactions']),
cnt['insert'], cnt['update'], cnt['payout'], cnt['upvote'],
accts, follows, int(ms), ' SLOW' if ms > 1000 else ''))
# once per hour, update accounts
if num % 1200 == 0:
Accounts.dirty_oldest(10000)
Accounts.flush(trx=True)
#Accounts.update_ranks()
# once a minute, update chain props
if num % 20 == 0:
update_chain_state()
def _process(cls, block, is_initial_sync=False):
"""Process a single block. Assumes a trx is open."""
# pylint: disable=too-many-boolean-expressions,too-many-branches
num = cls._push(block)
date = block['timestamp']
account_names = set()
comment_ops = []
json_ops = []
delete_ops = []
voted_authors = set()
for tx_idx, tx in enumerate(block['transactions']):
for operation in tx['operations']:
if isinstance(operation, dict):
op_type = operation['type'].split('_operation')[0]
op = operation['value']
else: # pre-appbase-style. remove after deploy. #APPBASE
op_type, op = operation
# account ops
if op_type == 'pow':
account_names.add(op['worker_account'])
elif op_type == 'pow2':
# old style. remove after #APPBASE
#account_names.add(op['work'][1]['input']['worker_account'])
account_names.add(op['work']['value']['input']['worker_account'])
elif op_type == 'account_create':
account_names.add(op['new_account_name'])
elif op_type == 'account_create_with_delegation':
account_names.add(op['new_account_name'])
# post ops
elif op_type == 'comment':
comment_ops.append(op)
elif op_type == 'delete_comment':
delete_ops.append(op)
elif op_type == 'vote':
if not is_initial_sync:
CachedPost.vote(op['author'], op['permlink'])
voted_authors.add(op['author']) # TODO: move to cachedpost
# misc ops
elif op_type == 'transfer':
Payments.op_transfer(op, tx_idx, num, date)
elif op_type == 'custom_json':
json_ops.append(op)
Accounts.register(account_names, date) # register any new names
Accounts.dirty(voted_authors) # update rep of voted authors
Posts.comment_ops(comment_ops, date) # handle inserts, edits
Posts.delete_ops(delete_ops) # handle post deletion
CustomOp.process_ops(json_ops, num, date) # follow/reblog/community ops
if (num > 20000000
and not account_names
and not voted_authors
and not comment_ops
and not delete_ops
and not json_ops):
trxs = len(block['transactions'])
print("[WARNING] block %d is no-op with %d trxs" % (num, trxs))
if trxs:
print("Trxs: {}".format(block['transactions']))
return num
def update(cls, op, date, pid):
# here we could also build content diffs...
if not DbState.is_initial_sync():
CachedPost.update(op['author'], op['permlink'], pid)
