def run(worker_name):
mongo = MongoStorage(db_name=os.getenv('DB_NAME', DB_NAME),
host=os.getenv('DB_HOST', MONGO_HOST),
port=os.getenv('DB_PORT', MONGO_PORT))
while True:
try:
if worker_name == 'scrape_operations':
mongo.ensure_indexes()
scrape_operations(mongo)
elif worker_name == 'validate_operations':
validate_operations(mongo)
elif worker_name == 'scrape_blockchain':
scrape_blockchain(mongo)
elif worker_name == 'scrape_all_users':
scrape_all_users(mongo, quick=False)
elif worker_name == 'scrape_prices':
scrape_prices(mongo)
elif worker_name == 'refresh_dbstats':
refresh_dbstats(mongo)
elif worker_name == 'override':
override(mongo)
else:
print(f'Worker {worker_name} does not exist!')
quit(1)
except (KeyboardInterrupt, SystemExit):
print('Quitting...')
exit(0)
except:
print('Exception in worker:', worker_name)
print(traceback.format_exc())
# prevent IO overflow
time.sleep(5)
def run(worker_name):
mongo = MongoStorage(
db_name=os.getenv('DB_NAME', DB_NAME),
host=os.getenv('DB_HOST', MONGO_HOST),
port=os.getenv('DB_PORT', MONGO_PORT))
while True:
try:
if worker_name == 'scrape_operations':
mongo.ensure_indexes()
scrape_operations(mongo)
elif worker_name == 'scrape_comments':
scrape_comments(mongo)
elif worker_name == 'post_processing':
post_processing(mongo)
elif worker_name == 'scrape_all_users':
scrape_all_users(mongo, quick=False)
elif worker_name == 'scrape_prices':
scrape_prices(mongo)
elif worker_name == 'refresh_dbstats':
refresh_dbstats(mongo)
else:
print(f'Worker "{worker_name}" does not exist!')
quit(1)
except (KeyboardInterrupt, SystemExit):
print('Quitting...')
exit(0)
except Exception as e:
print('Exception in worker:', worker_name)
log_exception()
time.sleep(5)
# prevent IO overflow
time.sleep(0.5)
def run_worker(worker_name):
mongo = MongoStorage(db_name=os.getenv('DB_NAME', DB_NAME),
host=os.getenv('DB_HOST', MONGO_HOST),
port=os.getenv('DB_PORT', MONGO_PORT))
while True:
try:
if worker_name == "scrape_operations":
mongo.ensure_indexes()
scrape_operations(mongo)
elif worker_name == "validate_operations":
validate_operations(mongo)
elif worker_name == "scrape_blockchain":
scrape_blockchain(mongo)
elif worker_name == "scrape_all_users":
scrape_all_users(mongo, quick=False)
elif worker_name == "scrape_all_users_quick":
scrape_all_users(mongo, quick=True)
elif worker_name == "scrape_prices":
scrape_prices(mongo)
elif worker_name == "refresh_dbstats":
refresh_dbstats(mongo)
elif worker_name == "override":
override(mongo)
except (KeyboardInterrupt, SystemExit):
print("Quitting...")
exit(0)
except:
print("EXCEPTION: %s():" % worker_name)
print(traceback.format_exc())
# prevent IO overflow
time.sleep(5)
.format('X' if api_status else ' '))
#print("| Python {}.x".format(py_version))
print("=================================================")
# Wait for user input
try:
i = read_usr_input('> ')#input("> ")
except:
clear_sreen()
i = '00'
continue
# Option 1: Scrape data from web pages
if i == '1':
print('Scraping data...')
data = scrape_prices(urls)
clear_sreen()
continue
# Option 2: Show scraped data
elif i == '2':
print('Scraped data:\n')
pp.pprint(data)
# Option 3: Store data in database
elif i == '3':
if data != None:
print('Storing data into MYSQL database.')
store_in_db(data)
else:
print('No data available to be stored, please scrape data first.')