def keywords_data():
"""情绪关键词数据
"""
customized = request.args.get('customized', '1')
query = request.args.get('query', None)
if query:
query = query.strip()
during = request.args.get('during', 24 * 3600)
during = int(during)
ts = request.args.get('ts', '')
ts = long(ts)
begin_ts = ts - during
end_ts = ts
limit = request.args.get('limit', 50)
limit = int(limit)
emotion = request.args.get('emotion', 'global')
results = {}
search_method = 'topic'
area = None
search_func = getattr(keywordsModule, 'search_%s_keywords' % search_method,
None)
if search_func:
if emotion == 'global':
keywords_data = {}
for k, v in emotions_kv.iteritems():
emotion_results = search_func(end_ts,
during,
v,
query=query,
domain=area,
top=limit,
customized=customized)
for keyword, count in emotion_results.iteritems():
try:
keywords_data[keyword] += count
except KeyError:
keywords_data[keyword] = count
kcount_tuple = sorted(keywords_data.iteritems(),
key=lambda (k, v): v,
reverse=False)
for k, v in kcount_tuple[len(kcount_tuple) - limit:]:
results[k] = v
else:
results[emotion] = search_func(end_ts,
during,
emotions_kv[emotion],
query=query,
domain=area,
top=limit,
customized=customized)
else:
return json.dumps('search function undefined')
return json.dumps(results)
def read_range_count_results(start_ts, over_ts, during=Hour):
over_ts = ts2HourlyTime(over_ts, MinInterval)
interval = (over_ts - start_ts) / during
emotion_dic = {}
if during <= MinInterval:
for k, v in emotions_kv.iteritems():
count = read_count_results(v, over_ts=over_ts, during=during)
emotion_dic[k] = [over_ts * 1000, count]
else:
end_ts = over_ts
start_ts = end_ts - during
for k, v in emotions_kv.iteritems():
count = read_count_results(v, start_ts=start_ts, over_ts=end_ts, during=during)
emotion_dic[k] = [end_ts * 1000, count]
return emotion_dic
def sentimentCronTopic(topic, xapian_search_weibo, start_ts, over_ts, sort_field=SORT_FIELD, save_fields=RESP_ITER_KEYS, \
during=Fifteenminutes, w_limit=TOP_WEIBOS_LIMIT, k_limit=TOP_KEYWORDS_LIMIT):
if topic and topic != '':
start_ts = int(start_ts)
over_ts = int(over_ts)
over_ts = ts2HourlyTime(over_ts, during)
interval = (over_ts - start_ts) / during
for i in range(interval, 0, -1):
emotions_kcount = {}
emotions_count = {}
emotions_weibo = {}
begin_ts = over_ts - during * i
end_ts = begin_ts + during
print begin_ts, end_ts, 'topic %s starts calculate' % topic.encode(
'utf-8')
query_dict = {
'timestamp': {
'$gt': begin_ts,
'$lt': end_ts
},
'$and': [
{
'$or': [{
'message_type': 1
}, {
'message_type': 3
}]
},
]
}
for k, v in emotions_kv.iteritems():
query_dict['sentiment'] = v
count, results = xapian_search_weibo.search(query=query_dict,
fields=save_fields)
mset = xapian_search_weibo.search(query=query_dict, sort_by=[sort_field], \
max_offset=w_limit, mset_direct=True)
kcount = top_keywords(gen_mset_iter(xapian_search_weibo,
mset,
fields=['terms']),
top=k_limit)
top_ws = top_weibos(results, top=w_limit)
emotions_count[v] = [end_ts, count]
emotions_kcount[v] = [end_ts, kcount]
emotions_weibo[v] = [end_ts, top_ws]
save_rt_results('count', topic, emotions_count, during)
save_rt_results('kcount', topic, emotions_kcount, during, k_limit,
w_limit)
save_rt_results('weibos', topic, emotions_weibo, during, k_limit,
w_limit)
def read_range_weibos_results(start_ts, over_ts, during=Hour):
over_ts = ts2HourlyTime(over_ts, MinInterval)
interval = (over_ts - start_ts) / during
emotion_dic = {}
if during <= MinInterval:
for k, v in emotions_kv.iteritems():
weibos = read_weibo_results(v, over_ts=over_ts, during=during)
emotion_dic[k] = weibos
else:
end_ts = over_ts
start_ts = end_ts - during
for k, v in emotions_kv.iteritems():
weibos = read_weibo_results(v, start_ts=start_ts, over_ts=end_ts, during=during)
emotion_dic[k] = weibos
return emotion_dic
def weibos_data():
"""关键微博
"""
customized = request.args.get('customized', '1')
query = request.args.get('query', None)
if query:
query = query.strip()
during = request.args.get('during', 24 * 3600)
during = int(during)
emotion = request.args.get('emotion', 'global')
ts = request.args.get('ts', '')
ts = long(ts)
begin_ts = ts - during
end_ts = ts
limit = request.args.get('limit', 50)
limit = int(limit)
results = {}
search_method = 'topic'
area = None
search_func = getattr(weibosModule, 'search_%s_weibos' % search_method,
None)
if search_func:
if emotion == 'global':
for k, v in emotions_kv.iteritems():
results[k] = search_func(end_ts,
during,
v,
query=query,
domain=area,
limit=limit,
customized=customized)
else:
results[emotion] = search_func(end_ts,
during,
emotions_kv[emotion],
query=query,
domain=area,
limit=limit,
customized=customized)
else:
return json.dumps('search function undefined')
return json.dumps(results)
def data():
"""分类情感数据--绝对值
"""
customized = request.args.get('customized', '1') # 该字段已舍弃
query = request.args.get('query', None) # 输入topic
if query:
query = query.strip()
during = request.args.get('during', 900) # 计算粒度，默认为15分钟
during = int(during)
ts = request.args.get('ts', '')
ts = long(ts)
begin_ts = ts - during
end_ts = ts
emotion = request.args.get('emotion', 'global') # 情绪类型
results = {}
search_method = 'topic'
area = None
search_func = getattr(countsModule, 'search_%s_counts' % search_method,
None)
if search_func:
if emotion == 'global':
for k, v in emotions_kv.iteritems():
results[k] = search_func(end_ts,
during,
v,
query=query,
domain=area,
customized=customized)
else:
results[emotion] = search_func(end_ts,
during,
emotions_kv[emotion],
query=query,
domain=area,
customized=customized)
else:
return json.dumps('search function undefined')
return json.dumps(results)
def sentimentCronTopic(topic, xapian_search_weibo, start_ts, over_ts, sort_field=SORT_FIELD, save_fields=RESP_ITER_KEYS, \
during=Fifteenminutes, w_limit=TOP_WEIBOS_LIMIT, k_limit=TOP_KEYWORDS_LIMIT):
if topic and topic != '':
start_ts = int(start_ts)
over_ts = int(over_ts)
over_ts = ts2HourlyTime(over_ts, during)
interval = (over_ts - start_ts) / during
for i in range(interval, 0, -1):
emotions_kcount = {}
emotions_count = {}
emotions_weibo = {}
begin_ts = over_ts - during * i
end_ts = begin_ts + during
print begin_ts, end_ts, 'topic %s starts calculate' % topic.encode('utf-8')
query_dict = {
'timestamp': {'$gt': begin_ts, '$lt': end_ts},
'$and': [
{'$or': [{'message_type': 1}, {'message_type': 3}]},
]
}
for k, v in emotions_kv.iteritems():
query_dict['sentiment'] = v
count, results = xapian_search_weibo.search(query=query_dict, fields=save_fields)
mset = xapian_search_weibo.search(query=query_dict, sort_by=[sort_field], \
max_offset=w_limit, mset_direct=True)
kcount = top_keywords(gen_mset_iter(xapian_search_weibo, mset, fields=['terms']), top=k_limit)
top_ws = top_weibos(results, top=w_limit)
emotions_count[v] = [end_ts, count]
emotions_kcount[v] = [end_ts, kcount]
emotions_weibo[v] = [end_ts, top_ws]
save_rt_results('count', topic, emotions_count, during)
save_rt_results('kcount', topic, emotions_kcount, during, k_limit, w_limit)
save_rt_results('weibos', topic, emotions_weibo, during, k_limit, w_limit)
def pie():
'''饼图数据
'''
query = request.args.get('query', None)
if query:
query = query.strip()
during = request.args.get('during', 1800)
during = int(during)
ts = request.args.get('ts', '')
ts = long(ts)
begin_ts = ts - during
end_ts = ts
emotion = request.args.get('emotion', 'global') # 情绪类型
results = {}
search_method = 'topic'
area = None
search_func = getattr(countsModule, 'search_%s_counts' % search_method,
None)
if search_func:
if emotion == 'global':
for k, v in emotions_kv.iteritems():
results[k] = search_func(end_ts,
during,
v,
query=query,
domain=area)[1]
else:
results[emotion] = search_func(end_ts,
during,
emotions_kv[emotion],
query=query,
domain=area)[1]
else:
return json.dumps('search function undefined')
return json.dumps(results)