def prepare_data(self, hash_): mem_key = self.get_memcache_key(hash_) data = self.redis.getvalue(mem_key) if data: return data year = int(self.get_argument('year')) timestamp = [year,] results = [] counts = dict(new_corps=0, total_corps=0, new_terminals=0, total_terminals=0) current_time = int(time.time()) for month in range(12): start_time, end_time = start_end_of_month(year=year, month=str(month+1)) if start_time > current_time: break total_corps = self.db.query("SELECT id FROM T_CORP WHERE timestamp <= %s", end_time) new_corps = self.db.query("SELECT id FROM T_CORP" " WHERE timestamp BETWEEN %s AND %s", start_time, end_time) total_terminals = self.db.query("SELECT id FROM T_TERMINAL_INFO WHERE begintime <= %s", end_time) new_terminals = self.db.query("SELECT id FROM T_TERMINAL_INFO" " WHERE begintime BETWEEN %s AND %s", start_time, end_time) result = dict(seq=month+1, new_corps=len(new_corps), total_corps=len(total_corps), new_terminals=len(new_terminals), total_terminals=len(total_terminals)) results.append(result) counts['new_corps'] += len(new_corps) counts['new_terminals'] += len(new_terminals) counts['total_corps'] = results[len(results)-1]['total_corps'] counts['total_terminals'] = results[len(results)-1]['total_terminals'] self.redis.setvalue(mem_key, (results, counts, timestamp), time=self.MEMCACHE_EXPIRY) return results, counts, timestamp
def retrieve(self, citylist=None, start_time=None, end_time=None): """core for Retrieving and Storing operation.""" #store the complete data of current month or last month into MongoDB if not(start_time and end_time): start_time, end_time = start_end_of_month() if not citylist: cities = self.mysql_db.query("SELECT DISTINCT region_code AS id FROM T_HLR_CITY") citylist = [c.id for c in cities] results = self.retrieve_mixin(citylist, start_time, end_time) try: for result in results: result = DotDict(result) oldresult = self.collection.find_one({'city_id': result.city_id, 'year': result.year, 'month': result.month}) if oldresult: result['_id'] = oldresult['_id'] else: result.pop('_id') self.collection.save(result) except: logging.exception('mongodb connected failed') return results