def run(worker_name):
mongo = MongoStorage(db_name=os.getenv('DB_NAME', DB_NAME),
host=os.getenv('DB_HOST', MONGO_HOST),
port=os.getenv('DB_PORT', MONGO_PORT))
while True:
try:
if worker_name == 'scrape_operations':
mongo.ensure_indexes()
scrape_operations(mongo)
elif worker_name == 'validate_operations':
validate_operations(mongo)
elif worker_name == 'scrape_blockchain':
scrape_blockchain(mongo)
elif worker_name == 'scrape_all_users':
scrape_all_users(mongo, quick=False)
elif worker_name == 'scrape_prices':
scrape_prices(mongo)
elif worker_name == 'refresh_dbstats':
refresh_dbstats(mongo)
elif worker_name == 'override':
override(mongo)
else:
print(f'Worker {worker_name} does not exist!')
quit(1)
except (KeyboardInterrupt, SystemExit):
print('Quitting...')
exit(0)
except:
print('Exception in worker:', worker_name)
print(traceback.format_exc())
# prevent IO overflow
time.sleep(5)
def run(worker_name):
mongo = MongoStorage(db_name=os.getenv('DB_NAME', DB_NAME),
host=os.getenv('DB_HOST', MONGO_HOST),
port=os.getenv('DB_PORT', MONGO_PORT))
while True:
try:
if worker_name == 'scrape_operations':
mongo.ensure_indexes()
scrape_operations(mongo)
elif worker_name == 'post_processing':
post_processing(mongo)
elif worker_name == 'scrape_all_users':
scrape_all_users(mongo, quick=False)
elif worker_name == 'refresh_dbstats':
refresh_dbstats(mongo)
else:
print(f'Worker "{worker_name}" does not exist!')
quit(1)
except (KeyboardInterrupt, SystemExit):
print('Quitting...')
exit(0)
except RuntimeError:
log_exception()
exit(1)
except Exception as e:
print('Exception in worker:', worker_name)
log_exception()
time.sleep(5)
# prevent IO overflow
time.sleep(0.5)
def run_worker(worker_name):
mongo = MongoStorage(db_name=os.getenv('DB_NAME', DB_NAME),
host=os.getenv('DB_HOST', MONGO_HOST),
port=os.getenv('DB_PORT', MONGO_PORT))
while True:
try:
if worker_name == "scrape_operations":
mongo.ensure_indexes()
scrape_operations(mongo)
elif worker_name == "validate_operations":
validate_operations(mongo)
elif worker_name == "scrape_blockchain":
scrape_blockchain(mongo)
elif worker_name == "scrape_all_users":
scrape_all_users(mongo, quick=False)
elif worker_name == "scrape_all_users_quick":
scrape_all_users(mongo, quick=True)
elif worker_name == "scrape_prices":
scrape_prices(mongo)
elif worker_name == "refresh_dbstats":
refresh_dbstats(mongo)
elif worker_name == "override":
override(mongo)
except (KeyboardInterrupt, SystemExit):
print("Quitting...")
exit(0)
except:
print("EXCEPTION: %s():" % worker_name)
print(traceback.format_exc())
# prevent IO overflow
time.sleep(5)