def update_account(mongo, username, load_extras=True):
""" Update Account.
If load_extras is True, update:
- followers, followings
- curation stats
- withdrawal routers, conversion requests
"""
a = Account(username)
account = {
**typify(a.export(load_extras=load_extras)),
'account': username,
'updatedAt': dt.datetime.utcnow(),
}
if type(account['json_metadata']) is dict:
account['json_metadata'] = \
strip_dot_from_keys(account['json_metadata'])
if not load_extras:
account = {'$set': account}
try:
mongo.Accounts.update({'name': a.name}, account, upsert=True)
except WriteError:
# likely an invalid profile
account['json_metadata'] = {}
mongo.Accounts.update({'name': a.name}, account, upsert=True)
print("Invalidated json_metadata on %s" % a.name)
def update_account_ops_quick(mongo, username, batch_size=200, steemd_instance=None):
""" Only update the latest history, limited to 1 batch of defined batch_size. """
start_index = account_operations_index(mongo, username)
# fetch latest records and update the db
history = \
Account(username,
steemd_instance=steemd_instance).history_reverse(batch_size=batch_size)
for event in take(batch_size, history):
if event['index'] < start_index:
return
with suppress(DuplicateKeyError):
mongo.AccountOperations.insert_one(json_expand(typify(event)))
def validate_operations(mongo):
""" Scan latest N blocks in db and validate its operations for consistency reasons. """
blockchain = Blockchain(mode="irreversible")
highest_block = mongo.Operations.find_one({}, sort=[('block_num', -1)])['block_num']
lowest_block = max(1, highest_block - 250_000)
for block_num in range(highest_block, lowest_block, -1):
if block_num % 100 == 0:
log.info('Validating block #%s' % block_num)
block = list(blockchain.stream(start_block=block_num, end_block=block_num))
# remove all invalid or changed operations
conditions = {'block_num': block_num, '_id': {'$nin': [x['_id'] for x in block]}}
mongo.Operations.delete_many(conditions)
# insert any missing operations
for op in block:
with suppress(DuplicateKeyError):
mongo.Operations.insert_one(json_expand(typify(op)))
# re-process comments (does not re-add deleted posts)
for comment in (x for x in block if x['type'] == 'comment'):
upsert_comment(mongo, '%s/%s' % (comment['author'], comment['permlink']))
def scrape_operations(mongo):
"""Fetch all operations from last known block forward."""
settings = Settings(mongo)
blockchain = Blockchain(mode="irreversible")
last_block = settings.last_block()
# handle batching
_batch_size = 50
_head_block_num = blockchain.get_current_block_num()
batch_dicts = []
history = blockchain.history(
start_block=last_block - _batch_size * 2,
)
def custom_merge(*args):
return list(set(filter(bool, flatten(args))))
def schedule_batch(_batch_dicts):
"""Send a batch to background worker, and reset _dicts container"""
_batch = merge_with(custom_merge, *_batch_dicts)
if _batch:
batch_update_async.delay(_batch)
log.info('\n> Fetching operations, starting with block %d...' % last_block)
for operation in history:
# handle comments
if operation['type'] in ['comment', 'delete_comment']:
post_identifier = "@%s/%s" % (operation['author'], operation['permlink'])
if operation['type'] == 'delete_comment':
delete_comment(mongo, post_identifier)
else:
# with suppress(TypeError):
upsert_comment(mongo, '%s/%s' % (operation['author'], operation['permlink']))
# update_comment_async.delay(post_identifier, recursive=True)
# if we're close to blockchain head, enable batching
recent_blocks = 20 * 60 * 24 * 1 # 1 days worth of blocks
if last_block > _head_block_num - recent_blocks:
batch_dicts.append(parse_operation(operation))
# insert operation
with suppress(DuplicateKeyError):
mongo.Operations.insert(json_expand(typify(operation)), check_keys=False)
# if this is a new block, checkpoint it, and schedule batch processing
if operation['block_num'] != last_block:
# print("last block", last_block)
last_block = operation['block_num']
settings.update_last_block(last_block - 1)
if last_block % 10 == 0:
_head_block_num = blockchain.get_current_block_num()
if last_block % _batch_size == 0:
schedule_batch(batch_dicts)
batch_dicts = []
if last_block % 100 == 0:
log.info("#%s: (%s)" % (
last_block,
blockchain.steem.hostname
))
def update_account_ops(mongo, username):
""" This method will fetch entire account history, and back-fill any missing ops. """
for event in Account(username).history():
with suppress(DuplicateKeyError):
mongo.AccountOperations.insert_one(json_expand(typify(event)))