def scrape_all_users(mongo, quick=False):
"""
Scrape all existing users
and insert/update their entries in Accounts collection.
"""
steem = Steem()
s = Settings(mongo)
quick = False # TODO: remove temporary override
account_checkpoint = s.account_checkpoint(quick)
if account_checkpoint:
usernames = list(get_usernames_batch(account_checkpoint, steem))
else:
usernames = list(get_usernames_batch(steem))
for username in usernames:
update_account(mongo, username, load_extras=quick)
if not quick:
update_account_ops(mongo, username)
s.set_account_checkpoint(username, quick)
log.info('Updated @%s' % username)
# this was the last batch
if account_checkpoint and len(usernames) < 1000:
s.set_account_checkpoint(-1, quick)
def scrape_all_users(mongo, quick=False):
"""
Scrape all existing users
and insert/update their entries in Accounts collection.
Ideally, this would only need to run once, because "scrape_accounts"
takes care of accounts that need to be updated in each block.
"""
steem = Steem()
indexer = Indexer(mongo)
account_checkpoint = indexer.get_checkpoint('accounts')
usernames = myAccounts()
# if account_checkpoint:
# usernames = list(get_usernames_batch(account_checkpoint, steem))
# else:
# usernames = list(get_usernames_batch(steem))
for username in usernames:
log.info('Updating @%s' % username)
update_account(mongo, username, load_extras=True)
if quick:
update_account_ops_quick(mongo, username)
else:
update_account_ops(mongo, username)
indexer.set_checkpoint('accounts', username)
log.info('Updated @%s' % username)
# this was the last batch
if account_checkpoint and len(usernames) < 1000:
indexer.set_checkpoint('accounts', -1)
def batch_update_async(batch_items: dict):
# todo break this batch into posts and account updates
# if we're lagging by a large margin, don't bother updating accounts
lag = time_delta(find_latest_item(mongo, 'Posts', 'created'))
if lag > 1000:
return
if use_multi_threading:
with log_exceptions():
thread_multi(
fn=update_account,
fn_args=[mongo, None],
dep_args=batch_items['accounts_light'],
fn_kwargs=dict(load_extras=False),
max_workers=num_threads,
)
thread_multi(
fn=update_account_ops_quick,
fn_args=[mongo, None],
dep_args=batch_items['accounts_light'],
fn_kwargs=None,
max_workers=num_threads,
)
else:
for account_name in batch_items['accounts_light']:
with log_exceptions():
update_account(mongo, account_name, load_extras=False)
update_account_ops_quick(mongo, account_name)
if use_multi_threading:
with log_exceptions():
thread_multi(
fn=update_account,
fn_args=[mongo, None],
dep_args=batch_items['accounts'],
fn_kwargs=dict(load_extras=True),
max_workers=num_threads,
)
thread_multi(
fn=update_account_ops_quick,
fn_args=[mongo, None],
dep_args=batch_items['accounts'],
fn_kwargs=None,
max_workers=num_threads,
)
else:
for account_name in batch_items['accounts']:
with log_exceptions():
update_account(mongo, account_name, load_extras=True)
update_account_ops_quick(mongo, account_name)
def scrape_all_users(mongo, steem=None):
"""Scrape all existing users and insert/update their entries in Accounts collection."""
s = Settings(mongo)
account_checkpoint = s.account_checkpoint()
if account_checkpoint:
usernames = list(get_usernames_batch(account_checkpoint, steem))
else:
usernames = list(get_usernames_batch(steem))
for username in usernames:
update_account(mongo, steem, username, load_extras=True)
update_account_ops(mongo, steem, username)
s.set_account_checkpoint(username)
print('Updated @%s' % username)
# this was the last batch
if account_checkpoint and len(usernames) < 1000:
s.set_account_checkpoint(-1)
def scrape_all_users(mongo, quick=False):
"""Scrape all existing users and insert/update their entries in Accounts collection."""
steem = Steem()
s = Settings(mongo)
account_checkpoint = s.account_checkpoint(quick)
if account_checkpoint:
usernames = list(get_usernames_batch(account_checkpoint, steem))
else:
usernames = list(get_usernames_batch(steem))
for username in usernames:
# Some errors in golos blockchain
with suppress(AccountDoesNotExistsException):
update_account(mongo, username, load_extras=quick)
if not quick:
update_account_ops(mongo, username)
s.set_account_checkpoint(username, quick)
log.info('Updated @%s' % username)
# this was the last batch
if account_checkpoint and len(usernames) < 1000:
s.set_account_checkpoint(-1, quick)
def batch_update_async(batch_items: dict):
# todo break this batch into posts and account updates
lag = time_delta(find_latest_item(mongo, 'Posts', 'created'))
if lag > 1000:
return
for identifier in batch_items['comments']:
with log_exceptions():
upsert_comment_chain(mongo, identifier, recursive=True)
# if we're lagging by a large margin, don't bother updating accounts
if lag > 500:
return
for account_name in batch_items['accounts_light']:
with log_exceptions():
update_account(mongo, account_name, load_extras=False)
update_account_ops_quick(mongo, account_name)
for account_name in batch_items['accounts']:
with log_exceptions():
update_account(mongo, account_name,
load_extras=True) # should be True
update_account_ops_quick(mongo, account_name)
def stream(BLOCK_NUM=None):
if BLOCK_NUM is None:
BLOCK_NUM = get_check_point('last_block') or get_current_block_num()
print(f'START STREAMING FROM BLOCK: {BLOCK_NUM}')
for op in blockchain.stream(start_block=BLOCK_NUM):
op_type = op['type']
def construct_identifier():
return '@%s/%s' % (
op.get('author', op.get('comment_author')),
op.get('permlink', op.get('comment_permlink')),
)
if op_type in ['account_create',
'create_claimed_account',
'account_create_with_delegation']:
update_account(op['creator'])
update_account(op['new_account_name'])
elif op_type in ['author_reward', 'comment']:
update_account(op['author'])
if op_type == 'comment':
swm_tag = get_swm_tag(op['body'])
if swm_tag:
try:
update_swm_post(construct_identifier(), swm_tag)
except NoSWMTag:
logging.exception('Update swm in parser err')
elif op_type == 'account_update':
update_account(op['account'])
# Update state
if op['block_num'] != BLOCK_NUM:
set_check_point('last_block', op['block_num'])
BLOCK_NUM = op['block_num']
def test():
m = MongoStorage()
with timeit():
update_account(m, Steem(), 'furion', load_extras=False)
def update_account_async(account_name):
update_account(mongo, stm, account_name, load_extras=False)
update_account_ops_quick(mongo, stm, account_name)