def post(self): dt = self.request.get("date", None) cursor = self.request.get("last_cursor", None) logging.info("date from request %s " % dt) if dt is None: date = datetime.datetime.now().date() - datetime.timedelta(days=1) logging.info("aggregatine users from yesterday.") else: date = datetime.datetime.strptime(dt, "%Y-%m-%d").date() if date >= datetime.datetime.now().date(): logging.info("too early , wait") self.response.out.write("too early . wait") return memcache_key_sessions = ( "sessions_for_date_" + str(datetime.datetime.today().date()) + "_" + str(date) + "_" + str(cursor) ) cached_sessions = memcache.get(memcache_key_sessions) if cached_sessions: logging.info("getting from cache for date %s" % str(date)) sessions = cached_sessions else: sessions = SessionModel.getDailyDataWithOffset(date, cursor) logging.info("session batch size %d" % len(sessions)) memcache.set(memcache_key_sessions, sessions) if sessions is None: logging.info("no sessions for date %s" % str(date)) return for s in sessions: memcache_key_s = "user_detail_" + str(datetime.datetime.now().date()) + "_" + str(date) + "_" + str(s.key()) if memcache.get(memcache_key_s): logging.info("skippin processed key %s for date %s" % (s.key(), str(date))) continue # links stats add to queue taskqueue.add( queue_name="data-consolidation", url="/aggregate_data", params={"sessionKey": s.key(), "upper_limit_date": date}, ) # TODO also create tas cue for user consolidation userStats = UserStats.gql("WHERE instapaper_account = :1 and date = :2", s.instaright_account, date).get() if userStats is None: logging.info("no user stats for user: %s and date: %s" % (s.instaright_account, str(date))) userStats = UserStats() userStats.instapaper_account = s.instaright_account userStats.count = 1 userStats.date = date userStats.put() else: logging.info("updating user stats for user %s and date %s" % (s.instaright_account, str(date))) userStats.count = userStats.count + 1 userStats.put() user_detail = UserDetails.gql("WHERE instapaper_account = :1", s.instaright_account).get() if user_detail is None: logging.info("new user: %s" % s.instaright_account) user_detail = UserDetails() user_detail.instapaper_account = s.instaright_account user_detail.last_active_date = s.date user_detail.put() # task queue that gathers info try: fetch_task_url = "/user/" + urllib2.quote(s.instaright_account) + "/fetch" except: logging.warn("can't fetch info for user %s " % s.instaright_account) logging.info("adding task on url %s" % fetch_task_url) taskqueue.add(queue_name="user-info", url=fetch_task_url) else: logging.info("updating usage for user: %s" % s.instaright_account) user_detail.last_active_date = s.date user_detail.links_added = user_detail.links_added + 1 user_detail.put() memcache.set(memcache_key_s, s.key()) # IMPORTANT:delete daily badges for user memcache.delete("badge_" + s.instaright_account) logging.info("done for date %s" % str(date)) self.response.out.write("done for date %s" % str(date))