def get_city_news(topic, start_ts, end_ts, unit=MinInterval, limit=TOP_NEWS_LIMIT):
weibos = []
if (end_ts - start_ts < unit):
upbound = int(math.ceil(end_ts / (unit * 1.0)) * unit)
item = db.session.query(CityNews).filter(CityNews.end==upbound, \
CityNews.topic==topic, \
CityNews.range==unit, \
CityNews.limit==limit).first()
if item:
news = _json_loads(item.news)
for news_item in news:
weibos.append((news_item['timestamp'],news_item))
else:
upbound = int(math.ceil(end_ts / (unit * 1.0)) * unit)
lowbound = (start_ts / unit) * unit
items = db.session.query(CityNews).filter(CityNews.end>lowbound, \
CityNews.end<=upbound, \
CityNews.topic==topic, \
CityNews.range==unit, \
CityNews.limit==limit).all()
for item in items:
news = _json_loads(item.news)
try:
for news_item in news:
weibos.append((news_item['timestamp'],news_item))
except:
continue
sorted_weibos = sorted(weibos, key=lambda k: k[0], reverse=True)
city_dict = {}
k = 0
for reposts_count, result in sorted_weibos:
k += 1
if (k > 1000):
break
"""
uid = result['user']
user_info = acquire_user_by_id(uid)
if user_info:
result['username'] = user_info['name']
else:
result['username'] = '******'
time = ts2date(result['timestamp'])
result['time'] = time
"""
try:
city = result['source_from_area'].split('\t')[1]
except:
city = ''
result['_id'] = deal_with(result['_id'])
if city in province_list:
try:
city_dict[city].append(result)
except:
city_dict[city] = [result]
return city_dict
def _top_weibos(weibos_dict, top=TOP_READ):
results_list = []
results_list_new = []
if weibos_dict != {}:
results = sorted(weibos_dict.iteritems(), key=lambda (k, v): v[0], reverse=False)
results = results[len(results) - top :]
for k, v in results:
results_list.extend(v[1])
for i in range(len(results_list)):
results_list_new.append(results_list[len(results_list) - 1 - i])
for r in results_list_new:
r["_id"] = deal_with(r["_id"])
return results_list_new
def _top_weibos(weibos_dict, top=TOP_READ):
results_list = []
results_list_new = []
if weibos_dict != {}:
results = sorted(weibos_dict.iteritems(),
key=lambda (k, v): v[0],
reverse=False)
results = results[len(results) - top:]
for k, v in results:
results_list.extend(v[1])
for i in range(len(results_list)):
results_list_new.append(results_list[len(results_list) - 1 - i])
for r in results_list_new:
r['_id'] = deal_with(r['_id'])
return results_list_new
def parseNews(news):
news_dict = {}
news = _json_loads(news)
if not news:
return {}
for weibo in news:
try:
_id = deal_with(weibo['_id'])
replies = 1
weibo['timestamp'] = ts2date(weibo['timestamp'])
weibo['content168'] = weibo['content168']
news_dict[_id] = [replies, weibo]
except:
continue
return news_dict
def get_city_news(topic,
start_ts,
end_ts,
unit=MinInterval,
limit=TOP_NEWS_LIMIT):
weibos = []
if (end_ts - start_ts < unit):
upbound = int(math.ceil(end_ts / (unit * 1.0)) * unit)
item = db.session.query(CityNews).filter(CityNews.end==upbound, \
CityNews.topic==topic, \
CityNews.range==unit, \
CityNews.limit==limit).first()
if item:
news = _json_loads(item.news)
for news_item in news:
weibos.append((news_item['timestamp'], news_item))
else:
upbound = int(math.ceil(end_ts / (unit * 1.0)) * unit)
lowbound = (start_ts / unit) * unit
items = db.session.query(CityNews).filter(CityNews.end>lowbound, \
CityNews.end<=upbound, \
CityNews.topic==topic, \
CityNews.range==unit, \
CityNews.limit==limit).all()
for item in items:
news = _json_loads(item.news)
try:
for news_item in news:
weibos.append((news_item['timestamp'], news_item))
except:
continue
sorted_weibos = sorted(weibos, key=lambda k: k[0], reverse=True)
city_dict = {}
k = 0
for reposts_count, result in sorted_weibos:
k += 1
if (k > 1000):
break
"""
uid = result['user']
user_info = acquire_user_by_id(uid)
if user_info:
result['username'] = user_info['name']
else:
result['username'] = '******'
time = ts2date(result['timestamp'])
result['time'] = time
"""
try:
city = result['source_from_area'].split('\t')[1]
except:
city = ''
result['_id'] = deal_with(result['_id'])
if city in province_list:
try:
city_dict[city].append(result)
except:
city_dict[city] = [result]
return city_dict