def prepare_data(self, hash_): mem_key = self.get_memcache_key(hash_) data = self.redis.getvalue(mem_key) if data: return data provinces = self.get_argument('province_id', None) cities = self.get_argument('city_id', None) start_time = int(self.get_argument('start_time', None)) end_time = int(self.get_argument('end_time', None)) group = self.get_argument('group', None) user_name = self.get_argument('user_name', None) mobile = self.get_argument('mobile', None) #if user_name and (not check_sql_injection(user_name)): # return [], [start_time, end_time] #if mobile and (not check_sql_injection(mobile)): # return [], [start_time, end_time] # the result of today is inavaliable d = datetime.datetime.fromtimestamp(time.time()) t = datetime.datetime.combine(datetime.date(d.year,d.month, d.day), datetime.time(0, 0, 0)) today_ = int(time.mktime(t.timetuple())) * 1000 if start_time >= today_: return [], [start_time, end_time] results = [] if not (cities or provinces): return results, [start_time, end_time] elif not cities: provinces = str_to_list(provinces) cities = city_list(provinces, self.db) else: cities = str_to_list(cities) cities = [int(c) for c in cities] cs = self.db.query("SELECT DISTINCT region_code FROM T_HLR_CITY" " WHERE city_id IN %s", tuple(cities + DUMMY_IDS)) citylist = [c.region_code for c in cs] query_term = {'city_id':{'$in':citylist}, 'fetchtime':{'$gte':start_time, '$lte':end_time}, 'group_id': None, 'user_name': None, 'mobile': None} for term in ['user_name', 'mobile']: v = self.get_argument(term, None) if v: query_term[term] = {'$regex': '\S*'+v+'\S*'} else: query_term.pop(term) if group: query_term['group_id'] = str(self.get_argument('group')) else: query_term.pop('group_id') display_term = {'province':1, 'city':1, 'group_name':1, 'user_name':1, 'mobile':1, '_id':0} res = [] try: mobiles = self.collection.find(query_term, {'mobile':1}).distinct('mobile') if mobiles: for m in mobiles: count = self.collection.find({'mobile':m, 'fetchtime':{'$gte':start_time, '$lte':end_time}}).count() r = self.collection.find_one({'mobile':m, 'fetchtime':{'$gte':start_time, '$lte':end_time}}, display_term) r['count'] = count res.append(r) if not res: ins = MSms() res = ins.retrieve(citylist, start_time, end_time) except: ins = MSmsMixin() res = ins.retrieve_mixin(citylist, start_time, end_time) res = ins.distinct(res) if user_name: p_name = re.compile(user_name) if mobile: p_mobile = re.compile(mobile) results = [] for r in res: flag = True if group and r['group_id'] != group: flag &= False if user_name and not p_name.findall(r['user_name']): flag &= False if mobile and not p_mobile.findall(r['mobile']): flag &= False if flag: results.append(r) self.redis.setvalue(mem_key,(results, [start_time, end_time]), time=self.MEMCACHE_EXPIRY) return results, [start_time, end_time]
def prepare_data(self, hash_): """Associated with the post method. workflow: if get value according the hash: return value else: retrieve the db and return the result. """ mem_key = self.get_memcache_key(hash_) data = self.redis.getvalue(mem_key) if data: return data # TODO: retrieve all the cities # HOW TO BEST PASS province and city? provinces = self.get_argument('province_id', None) cities = self.get_argument('city_id', None) start_time = int(self.get_argument('timestamp', None)) end_time = end_of_month(start_time) d = date.fromtimestamp(start_time/1000) year = d.year month = d.month # only last six days of this month can retrieve results s_time = start_of_month(int(time.time())*1000) e_time = end_of_month(int(time.time())*1000) if start_time == s_time: # this month cal = date.today().timetuple() year = cal.tm_year month = cal.tm_mon day = cal.tm_mday maxdays = monthrange(year, month)[1] if maxdays - day > ADMIN.MONTH_RETRIEVE_DAYS: return [], start_time elif start_time > e_time: return [], start_time if cities is None: provinces = str_to_list(provinces) cities = city_list(provinces, self.db) else: cities = str_to_list(cities) cities = [int(c) for c in cities] cs = self.db.query("SELECT DISTINCT region_code FROM T_HLR_CITY" " WHERE city_id IN %s", tuple(cities + DUMMY_IDS)) citylist = [int(c.region_code) for c in cs] query_term = {'city_id': {'$in': citylist}, 'year': year, 'month': month} try: results = list(self.collection.find(query_term, {'_id':0})) if not results: ins = MLocation() results = ins.retrieve(citylist, start_time, end_time) except: ins = MLocationMixin() results = ins.retrieve_mixin(citylist, start_time, end_time) self.redis.setvalue(mem_key, (results, start_time), time=self.MEMCACHE_EXPIRY) return results, start_time