def get(self, user): if user is None or len(user) == 0: logging.error('Empty user. Skipping') return user_decoded = urllib.unquote(user) logging.info('user: %s' %user_decoded) ud=UserDetails.gql('WHERE instapaper_account = :1' , user_decoded).get() if ud is None: logging.info('non existing user. redirect to home') self.redirect('/') return # sanity check if ud.instaright_account is None: ud.instaright_account = ud.instapaper_account ud.put() memcache_key ='user_info_' + user_decoded+'_'+str(datetime.datetime.now().date()) sessions = SessionModel.gql('WHERE instaright_account = :1 ORDER by date desc ' , user_decoded).fetch(100) links = [ s for s in sessions if s is not None ] cached_info = memcache.get(memcache_key) if cached_info: logging.info('getting from cache' ) template_variables = {'user':cached_info,'links':links} path= os.path.join(os.path.dirname(__file__), 'templates/user_info.html') self.response.headers["Content-type"] = "text/html" self.response.out.write(template.render(path,template_variables)) return user_detail= UserDetails.gql('WHERE mail = :1', user_decoded).get() if user_detail is None: logging.info('new user %s added to queue' %user_decoded) fetch_url = '/user/'+user+'/fetch' taskqueue.add(queue_name='user-info', url= fetch_url) ud = UserDetails() ud.name = user_decoded ud.instapaper_account = user_decoded ud.instaright_account = user_decoded ud.links_added = SessionModel.countAllForUser(user_decoded) # tmp put until we find more info for user ud.put() template_variables = {'user':ud, 'links': links} path= os.path.join(os.path.dirname(__file__), 'templates/user_info.html') self.response.headers["Content-type"] = "text/html" self.response.headers["Accept-Charset"] = "utf-8" self.response.out.write(template.render(path,template_variables)) return if user_detail.instaright_account is None: user_detail.instaright_account = user_decoded user_detail.put() memcache.set(memcache_key, user_detail) template_variables = {'user':user_detail, "links" : links} path= os.path.join(os.path.dirname(__file__), 'templates/user_info.html') self.response.headers["Content-type"] = "text/html" self.response.out.write(template.render(path,template_variables))
def post(self): dt = self.request.get("date", None) cursor = self.request.get("last_cursor", None) logging.info("date from request %s " % dt) if dt is None: date = datetime.datetime.now().date() - datetime.timedelta(days=1) logging.info("aggregatine users from yesterday.") else: date = datetime.datetime.strptime(dt, "%Y-%m-%d").date() if date >= datetime.datetime.now().date(): logging.info("too early , wait") self.response.out.write("too early . wait") return memcache_key_sessions = ( "sessions_for_date_" + str(datetime.datetime.today().date()) + "_" + str(date) + "_" + str(cursor) ) cached_sessions = memcache.get(memcache_key_sessions) if cached_sessions: logging.info("getting from cache for date %s" % str(date)) sessions = cached_sessions else: sessions = SessionModel.getDailyDataWithOffset(date, cursor) logging.info("session batch size %d" % len(sessions)) memcache.set(memcache_key_sessions, sessions) if sessions is None: logging.info("no sessions for date %s" % str(date)) return for s in sessions: memcache_key_s = "user_detail_" + str(datetime.datetime.now().date()) + "_" + str(date) + "_" + str(s.key()) if memcache.get(memcache_key_s): logging.info("skippin processed key %s for date %s" % (s.key(), str(date))) continue # links stats add to queue taskqueue.add( queue_name="data-consolidation", url="/aggregate_data", params={"sessionKey": s.key(), "upper_limit_date": date}, ) # TODO also create tas cue for user consolidation userStats = UserStats.gql("WHERE instapaper_account = :1 and date = :2", s.instaright_account, date).get() if userStats is None: logging.info("no user stats for user: %s and date: %s" % (s.instaright_account, str(date))) userStats = UserStats() userStats.instapaper_account = s.instaright_account userStats.count = 1 userStats.date = date userStats.put() else: logging.info("updating user stats for user %s and date %s" % (s.instaright_account, str(date))) userStats.count = userStats.count + 1 userStats.put() user_detail = UserDetails.gql("WHERE instapaper_account = :1", s.instaright_account).get() if user_detail is None: logging.info("new user: %s" % s.instaright_account) user_detail = UserDetails() user_detail.instapaper_account = s.instaright_account user_detail.last_active_date = s.date user_detail.put() # task queue that gathers info try: fetch_task_url = "/user/" + urllib2.quote(s.instaright_account) + "/fetch" except: logging.warn("can't fetch info for user %s " % s.instaright_account) logging.info("adding task on url %s" % fetch_task_url) taskqueue.add(queue_name="user-info", url=fetch_task_url) else: logging.info("updating usage for user: %s" % s.instaright_account) user_detail.last_active_date = s.date user_detail.links_added = user_detail.links_added + 1 user_detail.put() memcache.set(memcache_key_s, s.key()) # IMPORTANT:delete daily badges for user memcache.delete("badge_" + s.instaright_account) logging.info("done for date %s" % str(date)) self.response.out.write("done for date %s" % str(date))