def keywords_data(): """情绪关键词数据 """ customized = request.args.get('customized', '1') query = request.args.get('query', None) if query: query = query.strip() during = request.args.get('during', 24 * 3600) during = int(during) ts = request.args.get('ts', '') ts = long(ts) begin_ts = ts - during end_ts = ts limit = request.args.get('limit', 50) limit = int(limit) emotion = request.args.get('emotion', 'global') results = {} search_method = 'topic' area = None search_func = getattr(keywordsModule, 'search_%s_keywords' % search_method, None) if search_func: if emotion == 'global': keywords_data = {} for k, v in emotions_kv.iteritems(): emotion_results = search_func(end_ts, during, v, query=query, domain=area, top=limit, customized=customized) for keyword, count in emotion_results.iteritems(): try: keywords_data[keyword] += count except KeyError: keywords_data[keyword] = count kcount_tuple = sorted(keywords_data.iteritems(), key=lambda (k, v): v, reverse=False) for k, v in kcount_tuple[len(kcount_tuple) - limit:]: results[k] = v else: results[emotion] = search_func(end_ts, during, emotions_kv[emotion], query=query, domain=area, top=limit, customized=customized) else: return json.dumps('search function undefined') return json.dumps(results)
def read_range_count_results(start_ts, over_ts, during=Hour): over_ts = ts2HourlyTime(over_ts, MinInterval) interval = (over_ts - start_ts) / during emotion_dic = {} if during <= MinInterval: for k, v in emotions_kv.iteritems(): count = read_count_results(v, over_ts=over_ts, during=during) emotion_dic[k] = [over_ts * 1000, count] else: end_ts = over_ts start_ts = end_ts - during for k, v in emotions_kv.iteritems(): count = read_count_results(v, start_ts=start_ts, over_ts=end_ts, during=during) emotion_dic[k] = [end_ts * 1000, count] return emotion_dic
def sentimentCronTopic(topic, xapian_search_weibo, start_ts, over_ts, sort_field=SORT_FIELD, save_fields=RESP_ITER_KEYS, \ during=Fifteenminutes, w_limit=TOP_WEIBOS_LIMIT, k_limit=TOP_KEYWORDS_LIMIT): if topic and topic != '': start_ts = int(start_ts) over_ts = int(over_ts) over_ts = ts2HourlyTime(over_ts, during) interval = (over_ts - start_ts) / during for i in range(interval, 0, -1): emotions_kcount = {} emotions_count = {} emotions_weibo = {} begin_ts = over_ts - during * i end_ts = begin_ts + during print begin_ts, end_ts, 'topic %s starts calculate' % topic.encode( 'utf-8') query_dict = { 'timestamp': { '$gt': begin_ts, '$lt': end_ts }, '$and': [ { '$or': [{ 'message_type': 1 }, { 'message_type': 3 }] }, ] } for k, v in emotions_kv.iteritems(): query_dict['sentiment'] = v count, results = xapian_search_weibo.search(query=query_dict, fields=save_fields) mset = xapian_search_weibo.search(query=query_dict, sort_by=[sort_field], \ max_offset=w_limit, mset_direct=True) kcount = top_keywords(gen_mset_iter(xapian_search_weibo, mset, fields=['terms']), top=k_limit) top_ws = top_weibos(results, top=w_limit) emotions_count[v] = [end_ts, count] emotions_kcount[v] = [end_ts, kcount] emotions_weibo[v] = [end_ts, top_ws] save_rt_results('count', topic, emotions_count, during) save_rt_results('kcount', topic, emotions_kcount, during, k_limit, w_limit) save_rt_results('weibos', topic, emotions_weibo, during, k_limit, w_limit)
def read_range_weibos_results(start_ts, over_ts, during=Hour): over_ts = ts2HourlyTime(over_ts, MinInterval) interval = (over_ts - start_ts) / during emotion_dic = {} if during <= MinInterval: for k, v in emotions_kv.iteritems(): weibos = read_weibo_results(v, over_ts=over_ts, during=during) emotion_dic[k] = weibos else: end_ts = over_ts start_ts = end_ts - during for k, v in emotions_kv.iteritems(): weibos = read_weibo_results(v, start_ts=start_ts, over_ts=end_ts, during=during) emotion_dic[k] = weibos return emotion_dic
def weibos_data(): """关键微博 """ customized = request.args.get('customized', '1') query = request.args.get('query', None) if query: query = query.strip() during = request.args.get('during', 24 * 3600) during = int(during) emotion = request.args.get('emotion', 'global') ts = request.args.get('ts', '') ts = long(ts) begin_ts = ts - during end_ts = ts limit = request.args.get('limit', 50) limit = int(limit) results = {} search_method = 'topic' area = None search_func = getattr(weibosModule, 'search_%s_weibos' % search_method, None) if search_func: if emotion == 'global': for k, v in emotions_kv.iteritems(): results[k] = search_func(end_ts, during, v, query=query, domain=area, limit=limit, customized=customized) else: results[emotion] = search_func(end_ts, during, emotions_kv[emotion], query=query, domain=area, limit=limit, customized=customized) else: return json.dumps('search function undefined') return json.dumps(results)
def data(): """分类情感数据--绝对值 """ customized = request.args.get('customized', '1') # 该字段已舍弃 query = request.args.get('query', None) # 输入topic if query: query = query.strip() during = request.args.get('during', 900) # 计算粒度,默认为15分钟 during = int(during) ts = request.args.get('ts', '') ts = long(ts) begin_ts = ts - during end_ts = ts emotion = request.args.get('emotion', 'global') # 情绪类型 results = {} search_method = 'topic' area = None search_func = getattr(countsModule, 'search_%s_counts' % search_method, None) if search_func: if emotion == 'global': for k, v in emotions_kv.iteritems(): results[k] = search_func(end_ts, during, v, query=query, domain=area, customized=customized) else: results[emotion] = search_func(end_ts, during, emotions_kv[emotion], query=query, domain=area, customized=customized) else: return json.dumps('search function undefined') return json.dumps(results)
def sentimentCronTopic(topic, xapian_search_weibo, start_ts, over_ts, sort_field=SORT_FIELD, save_fields=RESP_ITER_KEYS, \ during=Fifteenminutes, w_limit=TOP_WEIBOS_LIMIT, k_limit=TOP_KEYWORDS_LIMIT): if topic and topic != '': start_ts = int(start_ts) over_ts = int(over_ts) over_ts = ts2HourlyTime(over_ts, during) interval = (over_ts - start_ts) / during for i in range(interval, 0, -1): emotions_kcount = {} emotions_count = {} emotions_weibo = {} begin_ts = over_ts - during * i end_ts = begin_ts + during print begin_ts, end_ts, 'topic %s starts calculate' % topic.encode('utf-8') query_dict = { 'timestamp': {'$gt': begin_ts, '$lt': end_ts}, '$and': [ {'$or': [{'message_type': 1}, {'message_type': 3}]}, ] } for k, v in emotions_kv.iteritems(): query_dict['sentiment'] = v count, results = xapian_search_weibo.search(query=query_dict, fields=save_fields) mset = xapian_search_weibo.search(query=query_dict, sort_by=[sort_field], \ max_offset=w_limit, mset_direct=True) kcount = top_keywords(gen_mset_iter(xapian_search_weibo, mset, fields=['terms']), top=k_limit) top_ws = top_weibos(results, top=w_limit) emotions_count[v] = [end_ts, count] emotions_kcount[v] = [end_ts, kcount] emotions_weibo[v] = [end_ts, top_ws] save_rt_results('count', topic, emotions_count, during) save_rt_results('kcount', topic, emotions_kcount, during, k_limit, w_limit) save_rt_results('weibos', topic, emotions_weibo, during, k_limit, w_limit)
def pie(): '''饼图数据 ''' query = request.args.get('query', None) if query: query = query.strip() during = request.args.get('during', 1800) during = int(during) ts = request.args.get('ts', '') ts = long(ts) begin_ts = ts - during end_ts = ts emotion = request.args.get('emotion', 'global') # 情绪类型 results = {} search_method = 'topic' area = None search_func = getattr(countsModule, 'search_%s_counts' % search_method, None) if search_func: if emotion == 'global': for k, v in emotions_kv.iteritems(): results[k] = search_func(end_ts, during, v, query=query, domain=area)[1] else: results[emotion] = search_func(end_ts, during, emotions_kv[emotion], query=query, domain=area)[1] else: return json.dumps('search function undefined') return json.dumps(results)