def updateWeeklyTop10Collection(top10, startDate, endDate):
if top10 != None and startDate != None and endDate != None:
if len(top10) > 0:
result = ttDB['week'].replace_one(
{'startDateStr': startDate['dateStr'], 'endDateStr': endDate['dateStr']},
{
'startDateStr': str(startDate['dateStr']),
'startDay': str(startDate['day']),
'startMonth': str(startDate['month']),
'startYear': str(startDate['year']),
'endDateStr': str(endDate['dateStr']),
'endDay': str(endDate['day']),
'endMonth': str(endDate['month']),
'endYear': str(endDate['year']),
'top10' : [
top10[0].getObject(),
top10[1].getObject(),
top10[2].getObject(),
top10[3].getObject(),
top10[4].getObject(),
top10[5].getObject(),
top10[6].getObject(),
top10[7].getObject(),
top10[8].getObject(),
top10[9].getObject()
]
},
True,
False
)
if result.acknowledged == False:
Utils.emitWarning([str(datetime.utcnow()),"Failed to save weekly top10. Acknowledgment was False."])
else:
Utils.emitWarning([str(datetime.utcnow()),"Params were not as expected when trying to save weekly top10."])
def updateMonthlyTop10Collection(top10, month, year):
if top10 != None and month != None and year != None:
if len(top10) > 0:
result = ttDB['month'].replace_one(
{'month': month, 'year': year},
{
'month': month,
'year': year,
'top10' : [
top10[0].getObject(),
top10[1].getObject(),
top10[2].getObject(),
top10[3].getObject(),
top10[4].getObject(),
top10[5].getObject(),
top10[6].getObject(),
top10[7].getObject(),
top10[8].getObject(),
top10[9].getObject()
]
},
True,
False
)
if result.acknowledged == False:
Utils.emitWarning([str(datetime.utcnow()),"Failed to save monthly top10. Acknowledgment was False."])
else:
Utils.emitWarning([str(datetime.utcnow()),"Params were not as expected when trying to save monthly top10."])
def updateAllTimeTop10Collection(top10):
if top10 != None:
if len(top10) > 0:
result = ttDB['alltime'].replace_one(
{'replace_key': 'AllTime'},
{
'replace_key': 'AllTime',
'top10' : [
top10[0].getObject(),
top10[1].getObject(),
top10[2].getObject(),
top10[3].getObject(),
top10[4].getObject(),
top10[5].getObject(),
top10[6].getObject(),
top10[7].getObject(),
top10[8].getObject(),
top10[9].getObject()
]
},
True,
False
)
if result.acknowledged == False:
Utils.emitWarning([str(datetime.utcnow()),"Failed to save alltime top10. Acknowledgment was False."])
else:
Utils.emitWarning([str(datetime.utcnow()),"Params were not as expected when trying to save alltime top10."])
def updateTrendsCollection(trends, date, woeid, city, country):
if date['dateStr'] != None and date['day'] != None and date['month'] != None and date['year'] != None and trends != None and woeid != None and city != None and country != None:
for trend in trends:
if trend['tweet_volume'] != None and date != "" and trend['name'] != None and trend['query'] != None and trend['url'] != None:
result = thDB['trending'].replace_one(
{'dateStr':str(date['dateStr']),'name':str(trend['name']),'city':str(city),'countryCode':str(country)},
{
'dateStr': str(date['dateStr']),
'day': str(date['day']),
'month': str(date['month']),
'year': str(date['year']),
'city': str(city),
'countryCode': str(country),
'name': str(trend['name']),
'tweet_volume': int(trend['tweet_volume']),
'query': str(trend['query']),
'url': str(trend['url'])
},
True,
False
)
if result.acknowledged == False:
Utils.emitWarning([str(datetime.utcnow()),"Failed to save trend. Acknowledgment was False."])
else:
Utils.emitWarning([str(datetime.utcnow()),"Params were not as expected when trying to save trends."])
def updateDailyTop10Collection(top10, date):
if date['dateStr'] != None and date['day'] != None and date['month'] != None and date['year'] != None and top10 != None:
if len(top10) > 0:
result = ttDB['day'].replace_one(
date,
{
'dateStr': str(date['dateStr']),
'day': str(date['day']),
'month': str(date['month']),
'year': str(date['year']),
'top10' : [
top10[0].getObject(),
top10[1].getObject(),
top10[2].getObject(),
top10[3].getObject(),
top10[4].getObject(),
top10[5].getObject(),
top10[6].getObject(),
top10[7].getObject(),
top10[8].getObject(),
top10[9].getObject()
]
},
True,
False
)
if result.acknowledged == False:
Utils.emitWarning([str(datetime.utcnow()),"Failed to save daily top10. Acknowledgment was False."])
else:
Utils.emitWarning([str(datetime.utcnow()),"Params were not as expected when trying to save daily top10."])
def updateDay(dateObj, utcToday, todaysTop10):
if utcToday != None:
dayTop10 = todaysTop10
#get date information for the new top10
dayTop10 = Top10Utils.getDateDataForSaving(dayTop10)
#save/update top10DB
TrendingTop10DB.updateDailyTop10Collection(dayTop10, dateObj)
else:
Utils.emitWarning([str(datetime.datetime.utcnow()),"No utcToday to use for generating Top10s."])
def getTop10(collection, dateQuery):
if dateQuery != None and collection != None:
result = ttDB[collection].find(dateQuery)
if result:
if result.count() == 0:
return False
else:
return result
else:
Utils.emitWarning([str(datetime.utcnow()),"Result was None or False when trying to find all trends for the date:",str(dateQuery)])
return False
def updateCitiesCollection(cities):
if cities != None:
for city in cities:
if city['countryCode'] == 'US' or city['countryCode'] == 'CA':
result = thDB['cities'].replace_one(
{'countryCode':str(city['countryCode']),'city':str(city['name']),'woeid':str(city['woeid'])},
{
'city': str(city['name']),
'countryCode': str(city['countryCode']),
'woeid': str(city['woeid'])
},
True,
False
)
if result.acknowledged == False:
Utils.emitWarning([str(datetime.utcnow()),"Failed to save city. Acknowledgment was False."])
else:
Utils.emitWarning([str(datetime.utcnow()),"Params were not as expected when trying to save cities"])
def getTrends(query, sort):
if query != None and sort == None:
result = thDB['trending'].find(query)
elif query != None and sort != None:
result = thDB['trending'].find(query).sort(sort)
elif query == None and sort != None:
result = thDB['trending'].find().sort(sort)
elif query == None and sort == None:
result = thDB['trending'].find()
if result:
if result.count() == 0:
Utils.emitWarning([str(datetime.utcnow()),"Returned result is of length zero."])
return False
else:
return result
else:
Utils.emitWarning([str(datetime.utcnow()),"Result was None or False when trying to find all trends for the query:",str(query)])
return False
def main():
#Initializing
logging.basicConfig(filename=Config.getTwendingTop10LoggingFile(), level=logging.WARNING)
con = True
while (con):
dateObj, utcToday = Utils.getDateObj()
todaysTrends = TrendingHistoryDB.getTrends(dateObj, None)
if todaysTrends:
todaysTrends = CollectionUtils.sortTrendsByName(todaysTrends)
todaysTop10 = Top10Utils.getTop10(todaysTrends)
todaysTrends = None
'''
Daily Top10 Update
'''
updateDay(dateObj, utcToday, todaysTop10)
'''
Weekly Top10 Update
'''
updateWeek(utcToday)
'''
Monthly Top10 Update
'''
updateMonth(utcToday)
'''
Yearly Top10 Update
'''
updateYear(utcToday)
'''
Alltime Top10 Update
'''
updateAllTime()
else:
Utils.emitWarning([str(datetime.datetime.utcnow()),"No trends to use for updating, DB return False."])
Utils.goToSleep(300)
Utils.emitWarning([str(datetime.datetime.utcnow()),"Main Loop Failed! Program Ending!"])
def updateAllTime():
data = TrendingTop10DB.getTop10('day',{})
if data != False:
top10Collection = []
if data != False and data[0]['top10']:
for top10 in data:
for item in top10['top10']:
obj = {
'name' : item['name'],
'count' : item['count'],
'url' : item['url']
}
top10Collection.append(obj)
if len(top10Collection) > 0:
allTimeTop10 = Top10Utils.createTop10List(top10Collection)
if allTimeTop10 and len(allTimeTop10) == 10:
#get date information for the new top10
allTimeTop10 = Top10Utils.getDateDataForSaving(allTimeTop10)
#save/update top10DB
TrendingTop10DB.updateAllTimeTop10Collection(allTimeTop10)
else:
Utils.emitWarning([str(datetime.datetime.utcnow()),"Top10 was either False or did not have 10 items exactly."])
else:
Utils.emitWarning([str(datetime.datetime.utcnow()),"Top10 Collection for month was empty."])
else:
Utils.emitWarning([str(datetime.datetime.utcnow()),"No day top10s were returned from DB. Cannot generate alltime top10."])
def updateYear(utcToday):
if utcToday != None:
currentYear = utcToday.strftime('%Y')
data = TrendingTop10DB.getTop10('day', {'year':str(currentYear)})
top10Collection = []
if data != False:
for top10 in data:
for item in top10['top10']:
obj = {
'name' : item['name'],
'count' : item['count'],
'url' : item['url']
}
top10Collection.append(obj)
if len(top10Collection) > 0:
yearTop10 = Top10Utils.createTop10List(top10Collection)
if yearTop10 and len(yearTop10) == 10:
#get date information for the new top10
yearTop10 = Top10Utils.getDateDataForSaving(yearTop10)
#save/update top10DB
TrendingTop10DB.updateYearlyTop10Collection(yearTop10, str(currentYear))
else:
Utils.emitWarning([str(datetime.datetime.utcnow()),"Top10 was either False or did not have 10 items exactly."])
else:
Utils.emitWarning([str(datetime.datetime.utcnow()),"Top10 Collection for month was empty."])
else:
Utils.emitWarning([str(datetime.datetime.utcnow()),"No utcToday to use for generating Top10s."])
def main():
#Initializing
logging.basicConfig(filename=Config.getTwendingHistoryLoggingFile(), level=logging.WARNING)
keys = Config.getTwitterKeys()
auth = OAuthHandler(keys['consumer_key'], keys['consumer_secret'])
auth.set_access_token(keys['access_token'], keys['access_token_secret'])
api = API(auth)
while (True):
dateObj,x = Utils.getDateObj()
x = None
if dateObj != None:
cities = TwitterAPI.getCities(api)
if cities != None:
TrendingHistoryDB.updateCitiesCollection(cities)
for city in cities:
if city['countryCode'] == 'US' or city['countryCode'] == 'CA':
if city['name'] != None and city['countryCode'] != None and city['woeid'] != None and dateObj != None and api != None:
trends = TwitterAPI.getTrends(city['woeid'], api)
if trends != None:
#Saving Trends
woeid = str(city['woeid'])
if (woeid != ""):
TrendingHistoryDB.updateTrendsCollection(trends, dateObj, woeid, city['name'], city['countryCode'])
else:
Utils.emitWarning([str(datetime.utcnow()),"Returned trends object was None Type."])
Utils.goToSleep(65)
else:
Utils.emitWarning([str(datetime.utcnow()),"Params were not as epectied before trying to get trends in given city."])
else:
Utils.emitWarning([str(datetime.utcnow()),"Cities list was None Type."])
break
else:
Utils.emitWarning([str(datetime.utcnow()),"DateObj in TrendingController is None Type."])
break
Utils.emitWarning([str(datetime.utcnow()),"Left the main loop, program terminating..."])
def updateWeek(utcToday):
if utcToday != None:
#calculate query
dayOfTheWeek = int(utcToday.strftime('%w'))
if dayOfTheWeek == 0:
endOfWeek = utcToday
startOfWeek = utcToday - datetime.timedelta(days=6)
else:
endOfWeek = utcToday + datetime.timedelta(days=(7 - int(dayOfTheWeek)))
startOfWeek = utcToday - datetime.timedelta(days=(int(dayOfTheWeek) - 1))
#create top10 for week
top10Collection = []
currentDate = startOfWeek
while(currentDate <= endOfWeek):
dateObj = Utils.getDateObjGivenDateTime(currentDate)
data = TrendingTop10DB.getTop10('day', dateObj)
if data != False and data[0]['top10']:
for item in data[0]['top10']:
obj = {
'name' : item['name'],
'count' : item['count'],
'url' : item['url']
}
top10Collection.append(obj)
currentDate = currentDate + datetime.timedelta(days=1)
if len(top10Collection) > 0:
weekTop10 = Top10Utils.createTop10List(top10Collection)
if weekTop10 and len(weekTop10) == 10:
#get date information for the new top10
weekTop10 = Top10Utils.getDateDataForSaving(weekTop10)
#save/update top10DB
TrendingTop10DB.updateWeeklyTop10Collection(weekTop10, Utils.getDateObjGivenDateTime(startOfWeek), Utils.getDateObjGivenDateTime(endOfWeek))
else:
Utils.emitWarning([str(datetime.datetime.utcnow()),"Top10 was either False or did not have 10 items exactly."])
else:
Utils.emitWarning([str(datetime.datetime.utcnow()),"Top10 Collection for week was empty."])
else:
Utils.emitWarning([str(datetime.datetime.utcnow()),"No utcToday to use for generating Top10s."])
def updateMonth(utcToday):
if utcToday != None:
currentMonth = utcToday.strftime('%m')
currentYear = utcToday.strftime('%Y')
monthStart = 1
temp, monthEnd = monthrange(int(currentYear), int(currentMonth))
top10Collection = []
for i in range(1, monthEnd + 1):
if i < 10: iStr = "0" + str(i)
else: iStr = str(i)
date = datetime.datetime.strptime("" + str(currentMonth) + " " + iStr + " " + str(currentYear), "%m %d %Y")
dateObj = Utils.getDateObjGivenDateTime(date)
data = TrendingTop10DB.getTop10('day', dateObj)
if data != False and data[0]['top10']:
for item in data[0]['top10']:
obj = {
'name' : item['name'],
'count' : item['count'],
'url' : item['url']
}
top10Collection.append(obj)
if len(top10Collection) > 0:
monthTop10 = Top10Utils.createTop10List(top10Collection)
if monthTop10 and len(monthTop10) == 10:
#get date information for the new top10
monthTop10 = Top10Utils.getDateDataForSaving(monthTop10)
#save/update top10DB
TrendingTop10DB.updateMonthlyTop10Collection(monthTop10, Utils.getMonthStr(currentMonth), str(currentYear))
else:
Utils.emitWarning([str(datetime.datetime.utcnow()),"Top10 was either False or did not have 10 items exactly."])
else:
Utils.emitWarning([str(datetime.datetime.utcnow()),"Top10 Collection for month was empty."])
else:
Utils.emitWarning([str(datetime.datetime.utcnow()),"No utcToday to use for generating Top10s."])
