def retrieve(self, citylist=None): """core for Retrieving and Storing operation.""" #store the complete data of curent month into MongoDB if not citylist: cities = self.mysql_db.query("SELECT DISTINCT region_code AS id" " FROM T_HLR_CITY") citylist = [c.id for c in cities] results = self.retrieve_mixin(citylist) # get the group_id list from mongodb(may include the group_id has been removed) res = self.collection.find({'city_id': { '$in' : citylist }}, {'group_id':1}) ids_mongod = [int(v['group_id']) for v in res] # get the group_id lsit from mysql (the latest) ids_mysql = [int(result['group_id']) for result in results] # get the group_id to be removed and remove them from mongodb ids_move = list(set(ids_mongod) - set(ids_mysql)) self.collection.remove({'group_id': {'$in':ids_move}}) try: for result in results: result = DotDict(result) oldresult = self.collection.find_one({'id': result.id}) if oldresult: result['_id'] = oldresult['_id'] else: result.pop('_id') self.collection.save(result) except: logging.exception('mongodb connected failed') return results
def retrieve(self, city_list=None, end_time=None): """core for Retrieving and Storing operation.""" """This method returns the last day""" if not end_time: d = datetime.datetime.fromtimestamp(time.time()) t = datetime.datetime.combine(datetime.date(d.year, d.month, d.day), datetime.time(0, 0)) # get today 0:00:00 end_time = int(time.mktime(t.timetuple())*1000) if not city_list: cities = self.mysql_db.query("SELECT DISTINCT region_code AS id FROM T_HLR_CITY") city_list = [city.id for city in cities] results = self.retrieve_mixin(city_list, end_time) try: for result in results: result = DotDict(result) query_term = {'city_id': result.city_id, 'timestamp': result.timestamp} oldresult = self.collection.find_one(query_term) if oldresult: result['_id'] = oldresult['_id'] else: result.pop('_id') self.collection.save(result) except Exception as e: logging.exception('mongodb saved failed. Error: %s', e.args) return results
def retrieve(self, citylist=None, start_time=None, end_time=None): """core for Retrieving and Storing operation.""" #store the complete data of current month or last month into MongoDB if not(start_time and end_time): start_time, end_time = start_end_of_month() if not citylist: cities = self.mysql_db.query("SELECT DISTINCT region_code AS id FROM T_HLR_CITY") citylist = [c.id for c in cities] results = self.retrieve_mixin(citylist, start_time, end_time) try: for result in results: result = DotDict(result) oldresult = self.collection.find_one({'city_id': result.city_id, 'year': result.year, 'month': result.month}) if oldresult: result['_id'] = oldresult['_id'] else: result.pop('_id') self.collection.save(result) except: logging.exception('mongodb connected failed') return results