def aggregateData(self, currentSession, upper_limit_date):
logging.info("agregate data for %s" % currentSession.date)
locationData = RequestUtils.ipResolverAPI(currentSession.ip)
if len(locationData) == 2:
logging.info("updating location data")
city = locationData[0]
countryCode = locationData[1]
# self.response.out.write('location api response:<BR> city : %s; country: %s ' % (locationData[1], locationData[3]))
logging.info("location api response: %s " % locationData)
userLocation = UserLocationModel()
userLocation.user = currentSession.instaright_account
userLocation.city = city
userLocation.countryCode = countryCode
userLocation.date = currentSession.date
userLocation.put()
# update country stats and city stats
logging.info("country update")
existingCountryStat = CountryStats.gql("WHERE countryCode = :1 ", countryCode).get()
if existingCountryStat:
# hack to avoid exception
if existingCountryStat.count is None:
existingCountryStat.count = 1
else:
existingCountryStat.count += 1
logging.info("updating count %s" % existingCountryStat.count)
existingCountryStat.dateUpdated = upper_limit_date
existingCountryStat.put()
else:
logging.info("new country")
countryStat = CountryStats()
countryStat.countryCode = countryCode
countryStat.count = 1
countryStat.count = upper_limit_date
countryStat.put()
logging.info("city update")
existingCityStat = CityStats.gql("WHERE city = :1 and countryCode = :2", city, countryCode).get()
if existingCityStat:
# hack to avoid exception
if existingCityStat.count is None:
existingCityStat.count = 1
else:
existingCityStat.count += 1
existingCityStat.dateUpdated = upper_limit_date
logging.info("updating count %s" % existingCityStat.count)
existingCityStat.put()
else:
logging.info("new city")
cityStat = CityStats()
cityStat.countryCode = countryCode
cityStat.city = city
cityStat.count = 1
cityStat.updateDate = upper_limit_date
cityStat.put()
existingLinkStat = LinkStats.gql("WHERE link = :1", currentSession.url).get()
logging.info("link stats update")
if existingLinkStat:
logging.info("new link %s" % currentSession.url)
existingLinkStat.count = existingLinkStat.count + 1
existingLinkStat.countUpdated = currentSession.date
existingLinkStat.lastUpdatedBy = currentSession.instaright_account
existingLinkStat.put()
else:
logging.info("updating link stats: %s" % currentSession.url)
linkStat = LinkStats()
linkStat.link = currentSession.url
linkStat.count = 1
linkStat.countUpdated = currentSession.date
linkStat.lastUpdatedBy = currentSession.instaright_account
linkStat.put()
# domain update
shouldUpdateSession = 0
mode = ""
if currentSession.domain is None or currentSession.domain == "":
currentSession.domain = RequestUtils.getDomain(currentSession.url)
shouldUpdateSession = 1
mode = "domain change: %s" % currentSession.domain
if currentSession.date is None or currentSession.date == "":
date = datetime.datetime.strptime("2009-11-15", "%Y-%m-%d").date()
currentSession.date = date
shouldUpdateSession = 2
mode = "date change: %s" % date
if shouldUpdateSession > 0:
logging.info("updating session mode: %s" % mode)
currentSession.put()
else:
logging.info("unchanged session")
logging.info("done data aggregation")
def updateStats(self, s):
dailyStats = DailyDomainStats.gql('WHERE domain = :1 and date = :2', s.domain, s.date).get()
if dailyStats is not None:
dailyStats.count += 1
logging.info('updating daily stats count %s' % dailyStats.count)
else:
logging.info('daily stats new domain: %s for date %s' %(s.domain, s.date))
dailyStats = DailyDomainStats()
dailyStats.domain = s.domain
dailyStats.count = 1
dailyStats.date = s.date.date()
dailyStats.put()
dow = s.date.weekday()
if dow == 0:
#mon
date = s.date + datetime.timedelta(days=1)
elif dow == 1:
#tue day of stats
date = s.date
elif dow == 2:
#wed check next tues
date = s.date + datetime.timedelta(days=6)
elif dow == 3:
#thu check next tues
date = s.date + datetime.timedelta(days=5)
elif dow == 4:
#fri check next tues
date = s.date + datetime.timedelta(days=4)
elif dow == 5:
#sat check next tues
date = s.date + datetime.timedelta(days=3)
elif dow == 6:
#sun check next tues
date = s.date + datetime.timedelta(days=2)
weeklyStats = WeeklyDomainStats.gql('WHERE domain = :1 and date = :2', s.domain, date).get()
if weeklyStats is not None:
weeklyStats.count += 1
logging.info('updating weekly stats count %s' % weeklyStats.count)
else:
logging.info('weekly stats new domain: %s for date %s' %(s.domain, str(date.date())))
weeklyStats = WeeklyDomainStats()
weeklyStats.domain = s.domain
weeklyStats.count = 1
weeklyStats.date = date.date()
#weeklyStats.put()
linkStats = LinkStats.gql('WHERE link = :1', s.url).get()
if linkStats is not None:
linkStats.count += 1
linkStats.countUpdated = datetime.datetime.now()
linkStats.lastUpdatedBy = s.instaright_account
logging.info('updated link stats %s' % linkStats.count)
else:
logging.info('new stat in link stats')
linkStats = LinkStats()
linkStats.count = 1
linkStats.link = s.url
linkStats.lastUpdatedBy = s.instaright_account
linkStats.put()
