def scrape_operations(mongo, steem=None):
"""Fetch all operations from last known block forward."""
settings = Settings(mongo)
blockchain = Blockchain(mode="irreversible", steem_instance=steem)
last_block = settings.last_block()
history = blockchain.replay(
start_block=last_block,
)
print('\n> Fetching operations, starting with block %d...' % last_block)
for operation in history:
# if operation is a main Post, update it in background
if operation['type'] == 'comment':
if not operation['parent_author'] and not is_comment(operation):
post_identifier = "@%s/%s" % (operation['author'], operation['permlink'])
update_post_async.delay(post_identifier)
# if we are up to date, trigger an update for referenced accounts
if last_block > blockchain.get_current_block_num() - 100:
for acc in extract_usernames_from_op(operation):
update_account_async.delay(acc)
# insert operation
with suppress(DuplicateKeyError):
mongo.Operations.insert_one(typify(operation))
# current block - 1 should become a new checkpoint
if operation['block_num'] != last_block:
last_block = operation['block_num']
settings.update_last_block(last_block - 1)
if last_block % 10 == 0:
print("#%s: %s" % (last_block, time.ctime()))
def update_account_ops_quick(mongo, steem, username):
start_index = account_operations_index(mongo, username)
# fetch latest records and update the db
for event in quick_history(steem, username, start_index):
with suppress(DuplicateKeyError):
mongo.AccountOperations.insert_one(typify(event))
def update_account(mongo, steem, username, load_extras=True):
""" Update Account. If load_extras """
a = Account(username, steem_instance=steem)
account = {
**typify(a.export(load_extras=load_extras)),
'account': username,
'updatedAt': dt.datetime.utcnow(),
}
if not load_extras:
account = {'$set': account}
mongo.Accounts.update({'name': a.name}, account, upsert=True)
def validate_operations(mongo, steem=None):
""" Scan each block in db and validate its operations for consistency reasons. """
blockchain = Blockchain(mode="irreversible", steem_instance=steem)
highest_block = mongo.Operations.find_one({}, sort=[('block_num', -1)])['block_num']
for block_num in range(highest_block, 1, -1):
if block_num % 10 == 0:
print('Validating block #%s' % block_num)
block = list(blockchain.stream(start=block_num, stop=block_num))
# remove all invalid or changed operations
conditions = {'block_num': block_num, '_id': {'$nin': [x['_id'] for x in block]}}
mongo.Operations.delete_many(conditions)
# insert any missing operations
for op in block:
with suppress(DuplicateKeyError):
mongo.Operations.insert_one(typify(op))
def test_typify():
untyped = {
'amount_to_sell': '0.784 SBD',
'block_num': 8926306,
'expiration': '2017-02-06T10:01:46',
'fill_or_kill': False,
'min_to_receive': '4.868 STEEM',
'proof': {
'inputs': [{
'foo': {
'bar': '4.868 STEEM'
}
}, 3055534, 16227194]
}
}
typed = {
'amount_to_sell': {
'amount': 0.784,
'asset': 'SBD'
},
'block_num': 8926306,
'expiration': datetime.datetime(2017, 2, 6, 10, 1, 46),
'fill_or_kill': False,
'min_to_receive': {
'amount': 4.868,
'asset': 'STEEM'
},
'proof': {
'inputs': [{
'foo': {
'bar': {
'amount': 4.868,
'asset': 'STEEM'
}
}
}, 3055534, 16227194]
}
}
assert typify(untyped) == typed
def update_account_ops(mongo, steem, username):
""" This method will fetch entire account history, and back-fill any missing ops. """
for event in Account(username, steem_instance=steem).history():
with suppress(DuplicateKeyError):
mongo.AccountOperations.insert_one(typify(event))