def retrieve(self, citylist=None):
"""core for Retrieving and Storing operation."""
#store the complete data of curent month into MongoDB
if not citylist:
cities = self.mysql_db.query("SELECT DISTINCT region_code AS id"
" FROM T_HLR_CITY")
citylist = [c.id for c in cities]
results = self.retrieve_mixin(citylist)
# get the group_id list from mongodb(may include the group_id has been removed)
res = self.collection.find({'city_id': { '$in' : citylist }}, {'group_id':1})
ids_mongod = [int(v['group_id']) for v in res]
# get the group_id lsit from mysql (the latest)
ids_mysql = [int(result['group_id']) for result in results]
# get the group_id to be removed and remove them from mongodb
ids_move = list(set(ids_mongod) - set(ids_mysql))
self.collection.remove({'group_id': {'$in':ids_move}})
try:
for result in results:
result = DotDict(result)
oldresult = self.collection.find_one({'id': result.id})
if oldresult:
result['_id'] = oldresult['_id']
else:
result.pop('_id')
self.collection.save(result)
except:
logging.exception('mongodb connected failed')
return results
def retrieve(self, city_list=None, end_time=None):
"""core for Retrieving and Storing operation."""
"""This method returns the last day"""
if not end_time:
d = datetime.datetime.fromtimestamp(time.time())
t = datetime.datetime.combine(datetime.date(d.year, d.month, d.day), datetime.time(0, 0))
# get today 0:00:00
end_time = int(time.mktime(t.timetuple())*1000)
if not city_list:
cities = self.mysql_db.query("SELECT DISTINCT region_code AS id FROM T_HLR_CITY")
city_list = [city.id for city in cities]
results = self.retrieve_mixin(city_list, end_time)
try:
for result in results:
result = DotDict(result)
query_term = {'city_id': result.city_id, 'timestamp': result.timestamp}
oldresult = self.collection.find_one(query_term)
if oldresult:
result['_id'] = oldresult['_id']
else:
result.pop('_id')
self.collection.save(result)
except Exception as e:
logging.exception('mongodb saved failed. Error: %s', e.args)
return results
def retrieve(self, citylist=None, start_time=None, end_time=None):
"""core for Retrieving and Storing operation."""
#store the complete data of current month or last month into MongoDB
if not(start_time and end_time):
start_time, end_time = start_end_of_month()
if not citylist:
cities = self.mysql_db.query("SELECT DISTINCT region_code AS id FROM T_HLR_CITY")
citylist = [c.id for c in cities]
results = self.retrieve_mixin(citylist, start_time, end_time)
try:
for result in results:
result = DotDict(result)
oldresult = self.collection.find_one({'city_id': result.city_id, 'year': result.year, 'month': result.month})
if oldresult:
result['_id'] = oldresult['_id']
else:
result.pop('_id')
self.collection.save(result)
except:
logging.exception('mongodb connected failed')
return results