def _process(cls, block, is_initial_sync=False):
"""Process a single block. Assumes a trx is open."""
#pylint: disable=too-many-branches
num = cls._push(block)
date = block['timestamp']
account_names = set()
json_ops = []
for tx_idx, tx in enumerate(block['transactions']):
for operation in tx['operations']:
op_type = operation['type']
op = operation['value']
# account ops
if op_type == 'pow_operation':
account_names.add(op['worker_account'])
elif op_type == 'pow2_operation':
account_names.add(
op['work']['value']['input']['worker_account'])
elif op_type == 'account_create_operation':
account_names.add(op['new_account_name'])
elif op_type == 'account_create_with_delegation_operation':
account_names.add(op['new_account_name'])
elif op_type == 'create_claimed_account_operation':
account_names.add(op['new_account_name'])
# account metadata updates
elif op_type == 'account_update_operation':
if not is_initial_sync:
Accounts.dirty(op['account']) # full
elif op_type == 'account_update2_operation':
if not is_initial_sync:
Accounts.dirty(op['account']) # full
# post ops
elif op_type == 'comment_operation':
Posts.comment_op(op, date)
if not is_initial_sync:
Accounts.dirty(op['author']) # lite - stats
elif op_type == 'delete_comment_operation':
Posts.delete_op(op)
elif op_type == 'vote_operation':
if not is_initial_sync:
Accounts.dirty(op['author']) # lite - rep
Accounts.dirty(op['voter']) # lite - stats
CachedPost.vote(op['author'], op['permlink'], None,
op['voter'])
# misc ops
elif op_type == 'transfer_operation':
Payments.op_transfer(op, tx_idx, num, date)
elif op_type == 'custom_json_operation':
json_ops.append(op)
Accounts.register(account_names, date) # register any new names
CustomOp.process_ops(json_ops, num,
date) # follow/reblog/community ops
return num
def _process(cls, block, is_initial_sync=False):
"""Process a single block. Assumes a trx is open."""
# pylint: disable=too-many-boolean-expressions,too-many-branches
num = cls._push(block)
date = block['timestamp']
account_names = set()
comment_ops = []
json_ops = []
delete_ops = []
voted_authors = set()
for tx_idx, tx in enumerate(block['transactions']):
for operation in tx['operations']:
if isinstance(operation, dict):
op_type = operation['type'].split('_operation')[0]
op = operation['value']
else: # pre-appbase-style. remove after deploy. #APPBASE
op_type, op = operation
# account ops
if op_type == 'pow':
account_names.add(op['worker_account'])
elif op_type == 'pow2':
# old style. remove after #APPBASE
#account_names.add(op['work'][1]['input']['worker_account'])
account_names.add(
op['work']['value']['input']['worker_account'])
elif op_type == 'account_create':
account_names.add(op['new_account_name'])
elif op_type == 'account_create_with_delegation':
account_names.add(op['new_account_name'])
# post ops
elif op_type == 'comment':
comment_ops.append(op)
elif op_type == 'delete_comment':
delete_ops.append(op)
elif op_type == 'vote':
if not is_initial_sync:
CachedPost.vote(op['author'], op['permlink'])
voted_authors.add(
op['author']) # TODO: move to cachedpost
# misc ops
elif op_type == 'transfer':
Payments.op_transfer(op, tx_idx, num, date)
elif op_type == 'custom_json':
json_ops.append(op)
Accounts.register(account_names, date) # register any new names
Accounts.dirty(voted_authors) # update rep of voted authors
Posts.comment_ops(comment_ops, date) # handle inserts, edits
Posts.delete_ops(delete_ops) # handle post deletion
CustomOp.process_ops(json_ops, num,
date) # follow/reblog/community ops
return num
def dirty_paidouts(cls, date):
paidout = cls._select_paidout_tuples(date)
authors = set()
for (pid, author, permlink) in paidout:
authors.add(author)
cls._dirty('payout', author, permlink, pid)
Accounts.dirty(authors) # force-update accounts on payout
if len(paidout) > 200:
print("[PREP] Found {} payouts for {} authors since {}".format(
len(paidout), len(authors), date))
return len(paidout)
def dirty_paidouts(cls, date):
"""Mark dirty all paidout posts not yet updated in db."""
paidout = cls._select_paidout_tuples(date)
authors = set()
for (pid, author, permlink) in paidout:
authors.add(author)
cls._dirty('payout', author, permlink, pid)
Accounts.dirty(authors) # force-update accounts on payout
if len(paidout) > 200:
log.info("[PREP] Found %d payouts for %d authors since %s",
len(paidout), len(authors), date)
return len(paidout)
def _process(cls, block, is_initial_sync=False):
num = cls._push(block)
date = block['timestamp']
account_names = set()
comment_ops = []
json_ops = []
delete_ops = []
voted_authors = set()
for tx_idx, tx in enumerate(block['transactions']):
for operation in tx['operations']:
op_type, op = operation
# account ops
if op_type == 'pow':
account_names.add(op['worker_account'])
elif op_type == 'pow2':
account_names.add(op['work'][1]['input']['worker_account'])
elif op_type == 'account_create':
account_names.add(op['new_account_name'])
elif op_type == 'account_create_with_delegation':
account_names.add(op['new_account_name'])
# post ops
elif op_type == 'comment':
comment_ops.append(op)
elif op_type == 'delete_comment':
delete_ops.append(op)
elif op_type == 'vote':
if not is_initial_sync:
CachedPost.vote(op['author'], op['permlink'])
voted_authors.add(
op['author']) # TODO: move to cachedpost
# misc ops
elif op_type == 'transfer':
Payments.op_transfer(op, tx_idx, num, date)
elif op_type == 'custom_json':
json_ops.append(op)
Accounts.register(account_names, date) # register any new names
Accounts.dirty(voted_authors) # update rep of voted authors
Posts.comment_ops(comment_ops, date) # handle inserts, edits
Posts.delete_ops(delete_ops) # handle post deletion
CustomOp.process_ops(json_ops, num,
date) # follow/reblog/community ops
return num
def vote(cls, author, permlink, pid=None):
"""Handle a post dirtied by a `vote` op."""
cls._dirty('upvote', author, permlink, pid)
Accounts.dirty(set([author])) # rep changed
def process_block(block, is_initial_sync=False):
date = block['timestamp']
block_id = block['block_id']
prev = block['previous']
block_num = int(block_id[:8], base=16)
txs = block['transactions']
query(
"INSERT INTO hive_blocks (num, hash, prev, txs, created_at) "
"VALUES (:num, :hash, :prev, :txs, :date)",
num=block_num,
hash=block_id,
prev=prev,
txs=len(txs),
date=date)
accounts = set()
comments = []
json_ops = []
deleted = []
dirty = set()
for tx in txs:
for operation in tx['operations']:
op_type, op = operation
if op_type == 'pow':
accounts.add(op['worker_account'])
elif op_type == 'pow2':
accounts.add(op['work'][1]['input']['worker_account'])
elif op_type in [
'account_create', 'account_create_with_delegation'
]:
accounts.add(op['new_account_name'])
elif op_type == 'comment':
comments.append(op)
dirty.add(op['author'] + '/' + op['permlink'])
Accounts.dirty(op['author'])
if op['parent_author']:
Accounts.dirty(op['parent_author'])
elif op_type == 'delete_comment':
deleted.append(op)
elif op_type == 'custom_json':
json_ops.append(op)
elif op_type == 'vote':
dirty.add(op['author'] + '/' + op['permlink'])
Accounts.dirty(op['author'])
Accounts.dirty(op['voter'])
Accounts.register(
accounts,
date) # if an account does not exist, mark it as created in this block
Posts.register(
comments, date
) # if this is a new post, add the entry and validate community param
Posts.delete(deleted) # mark hive_posts.is_deleted = 1
for op in json_ops:
if op['id'] not in ['follow', 'com.steemit.community']:
continue
# we are assuming `required_posting_auths` is always used and length 1.
# it may be that some ops will require `required_active_auths` instead
# (e.g. if we use that route for admin action of acct creation)
# if op['required_active_auths']:
# log.warning("unexpected active auths: %s" % op)
if len(op['required_posting_auths']) != 1:
log.warning("unexpected auths: %s" % op)
continue
account = op['required_posting_auths'][0]
op_json = load_json_key(op, 'json')
if op['id'] == 'follow':
if block_num < 6000000 and type(op_json) != list:
op_json = ['follow', op_json] # legacy compat
process_json_follow_op(account, op_json, date)
elif op['id'] == 'com.steemit.community':
if block_num > 13e6:
process_json_community_op(account, op_json, date)
# return all posts modified this block
return dirty