def analyzeStocks(date, stocks):
dateString = date.strftime("%Y-%m-%d")
for symbol in stocks:
print(symbol)
db = clientStockTweets.get_database('stocks_data_db')
(shouldParse, hours) = shouldParseStock(symbol, dateString)
if (shouldParse is False):
continue
(soup, errorMsg, timeElapsed) = findPageStock(symbol, hours)
if (soup == ''):
stockError = {
'date': dateString,
'symbol': symbol,
'error': errorMsg,
'timeElapsed': timeElapsed
}
db.stock_tweets_errors.insert_one(stockError)
continue
try:
result = parseStockData(symbol, soup)
except Exception as e:
stockError = {
'date': dateString,
'symbol': symbol,
'error': str(e),
'timeElapsed': -1
}
db.stock_tweets_errors.insert_one(stockError)
print(e)
continue
if (len(result) == 0):
stockError = {
'date': dateString,
'symbol': symbol,
'error': 'Result length is 0??',
'timeElapsed': -1
}
db.stock_tweets_errors.insert_one(stockError)
print(stockError)
continue
results = updateLastMessageTime(db, symbol, result)
# No new messages
if (len(results) != 0):
insertResults(results)
updateLastParsedTime(db, symbol)
def dailyAnalyzeUsers(reAnalyze, updateUser, daysback):
users = parseOldUsers(daysback)
print(len(users))
for username in users:
print(username)
coreInfo = shouldParseUser(username, reAnalyze, updateUser)
if (not coreInfo):
continue
(soup, errorMsg, timeElapsed) = findPageUser(username)
coreInfo['timeElapsed'] = timeElapsed
if (errorMsg != ''):
coreInfo['error'] = errorMsg
insertUpdateError(coreInfo, reAnalyze, updateUser)
continue
result = parseUserData(username, soup)
if (len(result) == 0):
coreInfo['error'] = "Empty result list"
insertUpdateError(coreInfo, reAnalyze, updateUser)
continue
insertUpdateError(coreInfo, reAnalyze, updateUser)
insertResults(result)