def aggregateData(self, currentSession, upper_limit_date): logging.info("agregate data for %s" % currentSession.date) locationData = RequestUtils.ipResolverAPI(currentSession.ip) if len(locationData) == 2: logging.info("updating location data") city = locationData[0] countryCode = locationData[1] # self.response.out.write('location api response:<BR> city : %s; country: %s ' % (locationData[1], locationData[3])) logging.info("location api response: %s " % locationData) userLocation = UserLocationModel() userLocation.user = currentSession.instaright_account userLocation.city = city userLocation.countryCode = countryCode userLocation.date = currentSession.date userLocation.put() # update country stats and city stats logging.info("country update") existingCountryStat = CountryStats.gql("WHERE countryCode = :1 ", countryCode).get() if existingCountryStat: # hack to avoid exception if existingCountryStat.count is None: existingCountryStat.count = 1 else: existingCountryStat.count += 1 logging.info("updating count %s" % existingCountryStat.count) existingCountryStat.dateUpdated = upper_limit_date existingCountryStat.put() else: logging.info("new country") countryStat = CountryStats() countryStat.countryCode = countryCode countryStat.count = 1 countryStat.count = upper_limit_date countryStat.put() logging.info("city update") existingCityStat = CityStats.gql("WHERE city = :1 and countryCode = :2", city, countryCode).get() if existingCityStat: # hack to avoid exception if existingCityStat.count is None: existingCityStat.count = 1 else: existingCityStat.count += 1 existingCityStat.dateUpdated = upper_limit_date logging.info("updating count %s" % existingCityStat.count) existingCityStat.put() else: logging.info("new city") cityStat = CityStats() cityStat.countryCode = countryCode cityStat.city = city cityStat.count = 1 cityStat.updateDate = upper_limit_date cityStat.put() existingLinkStat = LinkStats.gql("WHERE link = :1", currentSession.url).get() logging.info("link stats update") if existingLinkStat: logging.info("new link %s" % currentSession.url) existingLinkStat.count = existingLinkStat.count + 1 existingLinkStat.countUpdated = currentSession.date existingLinkStat.lastUpdatedBy = currentSession.instaright_account existingLinkStat.put() else: logging.info("updating link stats: %s" % currentSession.url) linkStat = LinkStats() linkStat.link = currentSession.url linkStat.count = 1 linkStat.countUpdated = currentSession.date linkStat.lastUpdatedBy = currentSession.instaright_account linkStat.put() # domain update shouldUpdateSession = 0 mode = "" if currentSession.domain is None or currentSession.domain == "": currentSession.domain = RequestUtils.getDomain(currentSession.url) shouldUpdateSession = 1 mode = "domain change: %s" % currentSession.domain if currentSession.date is None or currentSession.date == "": date = datetime.datetime.strptime("2009-11-15", "%Y-%m-%d").date() currentSession.date = date shouldUpdateSession = 2 mode = "date change: %s" % date if shouldUpdateSession > 0: logging.info("updating session mode: %s" % mode) currentSession.put() else: logging.info("unchanged session") logging.info("done data aggregation")
def updateStats(self, s): dailyStats = DailyDomainStats.gql('WHERE domain = :1 and date = :2', s.domain, s.date).get() if dailyStats is not None: dailyStats.count += 1 logging.info('updating daily stats count %s' % dailyStats.count) else: logging.info('daily stats new domain: %s for date %s' %(s.domain, s.date)) dailyStats = DailyDomainStats() dailyStats.domain = s.domain dailyStats.count = 1 dailyStats.date = s.date.date() dailyStats.put() dow = s.date.weekday() if dow == 0: #mon date = s.date + datetime.timedelta(days=1) elif dow == 1: #tue day of stats date = s.date elif dow == 2: #wed check next tues date = s.date + datetime.timedelta(days=6) elif dow == 3: #thu check next tues date = s.date + datetime.timedelta(days=5) elif dow == 4: #fri check next tues date = s.date + datetime.timedelta(days=4) elif dow == 5: #sat check next tues date = s.date + datetime.timedelta(days=3) elif dow == 6: #sun check next tues date = s.date + datetime.timedelta(days=2) weeklyStats = WeeklyDomainStats.gql('WHERE domain = :1 and date = :2', s.domain, date).get() if weeklyStats is not None: weeklyStats.count += 1 logging.info('updating weekly stats count %s' % weeklyStats.count) else: logging.info('weekly stats new domain: %s for date %s' %(s.domain, str(date.date()))) weeklyStats = WeeklyDomainStats() weeklyStats.domain = s.domain weeklyStats.count = 1 weeklyStats.date = date.date() #weeklyStats.put() linkStats = LinkStats.gql('WHERE link = :1', s.url).get() if linkStats is not None: linkStats.count += 1 linkStats.countUpdated = datetime.datetime.now() linkStats.lastUpdatedBy = s.instaright_account logging.info('updated link stats %s' % linkStats.count) else: logging.info('new stat in link stats') linkStats = LinkStats() linkStats.count = 1 linkStats.link = s.url linkStats.lastUpdatedBy = s.instaright_account linkStats.put()