def scrape_all_users(mongo, quick=False):
"""
Scrape all existing users
and insert/update their entries in Accounts collection.
Ideally, this would only need to run once, because "scrape_accounts"
takes care of accounts that need to be updated in each block.
"""
steem = Steem()
indexer = Indexer(mongo)
account_checkpoint = indexer.get_checkpoint('accounts')
usernames = myAccounts()
# if account_checkpoint:
# usernames = list(get_usernames_batch(account_checkpoint, steem))
# else:
# usernames = list(get_usernames_batch(steem))
for username in usernames:
log.info('Updating @%s' % username)
update_account(mongo, username, load_extras=True)
if quick:
update_account_ops_quick(mongo, username)
else:
update_account_ops(mongo, username)
indexer.set_checkpoint('accounts', username)
log.info('Updated @%s' % username)
# this was the last batch
if account_checkpoint and len(usernames) < 1000:
indexer.set_checkpoint('accounts', -1)
def scrape_all_users(mongo, quick=False):
"""
Scrape all existing users
and insert/update their entries in Accounts collection.
"""
steem = Steem()
s = Settings(mongo)
quick = False # TODO: remove temporary override
account_checkpoint = s.account_checkpoint(quick)
if account_checkpoint:
usernames = list(get_usernames_batch(account_checkpoint, steem))
else:
usernames = list(get_usernames_batch(steem))
for username in usernames:
update_account(mongo, username, load_extras=quick)
if not quick:
update_account_ops(mongo, username)
s.set_account_checkpoint(username, quick)
log.info('Updated @%s' % username)
# this was the last batch
if account_checkpoint and len(usernames) < 1000:
s.set_account_checkpoint(-1, quick)
def scrape_all_users(mongo, steem=None):
"""Scrape all existing users and insert/update their entries in Accounts collection."""
s = Settings(mongo)
account_checkpoint = s.account_checkpoint()
if account_checkpoint:
usernames = list(get_usernames_batch(account_checkpoint, steem))
else:
usernames = list(get_usernames_batch(steem))
for username in usernames:
update_account(mongo, steem, username, load_extras=True)
update_account_ops(mongo, steem, username)
s.set_account_checkpoint(username)
print('Updated @%s' % username)
# this was the last batch
if account_checkpoint and len(usernames) < 1000:
s.set_account_checkpoint(-1)
def scrape_all_users(mongo, quick=False):
"""Scrape all existing users and insert/update their entries in Accounts collection."""
steem = Steem()
s = Settings(mongo)
account_checkpoint = s.account_checkpoint(quick)
if account_checkpoint:
usernames = list(get_usernames_batch(account_checkpoint, steem))
else:
usernames = list(get_usernames_batch(steem))
for username in usernames:
# Some errors in golos blockchain
with suppress(AccountDoesNotExistsException):
update_account(mongo, username, load_extras=quick)
if not quick:
update_account_ops(mongo, username)
s.set_account_checkpoint(username, quick)
log.info('Updated @%s' % username)
# this was the last batch
if account_checkpoint and len(usernames) < 1000:
s.set_account_checkpoint(-1, quick)