def utils_get_influence(wxbot_id, period, startdate, enddate): start_ts, end_ts, period = dump_date(period, startdate, enddate) current_timestamp = int(time.time()) current_date = ts2datetime(current_timestamp) if period == 0: #获取今天的数据 xnr_puid = load_wxxnr_redis_data(wxbot_id=wxbot_id, items=['puid'])['puid'] current_time = datetime2ts(current_date) query_at_num = { 'query': { 'bool': { 'must': [{ 'term': { 'xnr_id': xnr_puid } }, { 'term': { 'at_flag': 1 } }] } } } #虚拟人今天被@数量 at_num_xnr = 0 wx_group_message_index_name = wx_group_message_index_name_pre + current_date try: results_xnr = es_xnr.count(index=wx_group_message_index_name, doc_type=wx_group_message_index_type, body=query_at_num) if results_xnr['_shards']['successful'] != 0: at_num_xnr = results_xnr['count'] except Exception, e: print 'at_num_xnr Exception: ', str(e) # 截止目前所有被@总数 at_num_total = 0 wx_group_message_index_list = get_wx_groupmessage_index_list( WX_GROUP_MESSAGE_START_DATE_ASSESSMENT, ts2datetime(current_time)) for index_name in wx_group_message_index_list: try: r = es_xnr.count(index=index_name, doc_type=wx_group_message_index_type, body=query_at_num) if r['_shards']['successful'] != 0: at_num_total += r['count'] except Exception, e: pass
def wx_history_count(xnr_user_no, xnr_puid, current_time): current_date = ts2datetime(current_time) last_date = ts2datetime(current_time - DAY) #获取今日发言总数 today_count = 0 query_body = { 'query': { 'bool': { 'must': [{ 'term': { 'speaker_id': xnr_puid } }, { 'term': { 'xnr_id': xnr_puid } }] } } } today_index_name = wx_group_message_index_name_pre + current_date try: today_count_result = es_xnr.count(index=today_index_name, doc_type=wx_group_message_index_type, body=query_body) if today_count_result['_shards']['successful'] != 0: today_count = today_count_result['count'] except Exception, e: print 'today_count Exception: ', str(e)
def get_influence_at_num(puid): current_timestamp = int(time.time()-DAY) current_date = ts2datetime(current_timestamp) current_time = datetime2ts(current_date) query_at_num = { 'query':{ 'bool':{ 'must':[ {'term':{'xnr_id':puid}}, {'term':{'at_flag':1}} ] } } } #虚拟人今天被@数量 at_num_xnr = 0 wx_group_message_index_name = wx_group_message_index_name_pre + current_date try: results_xnr = es_xnr.count(index=wx_group_message_index_name,doc_type=wx_group_message_index_type,body=query_at_num) if results_xnr['_shards']['successful'] != 0: at_num_xnr = results_xnr['count'] except Exception,e: #print 'at_num_xnr Exception: ', str(e) pass
def utils_get_safe(wxbot_id, period, startdate, enddate): start_ts, end_ts, period = dump_date(period, startdate, enddate) current_timestamp = int(time.time()) current_date = ts2datetime(current_timestamp) if period == 0: #获取今天的数据 current_time = datetime2ts(current_date) last_date = ts2datetime(current_time-DAY) speak_dict = {} speak_dict['speak_day'] = {} speak_dict['speak_total'] = {} xnr_puid = load_wxxnr_redis_data(wxbot_id=wxbot_id, items=['puid'])['puid'] #获取xnr今日发言总数 today_count = 0 query_body = { 'query':{ 'bool':{ 'must':[ {'term':{'speaker_id': xnr_puid}}, {'term':{'xnr_id':xnr_puid}} ] } } } today_index_name = wx_group_message_index_name_pre + current_date try: today_count_result = es_xnr.count(index=today_index_name,doc_type=wx_group_message_index_type,body=query_body) if today_count_result['_shards']['successful'] != 0: today_count = today_count_result['count'] except Exception,e: pass #print 'today_count Exception: ', str(e) #获取xnr历史发言总数 total_count = 0 total_query_body = { 'query':{ 'bool':{ 'must':[ {'term':{'xnr_user_no': wxbot_id}}, {'term':{'puid':xnr_puid}}, {'term':{'date_time':last_date}} ] } } } total_index_name = wx_xnr_history_count_index_name try: total_count_result = es_xnr.search(index=total_index_name,doc_type=wx_xnr_history_count_index_type,body=total_query_body) if total_count_result['_shards']['successful'] != 0: total_count = total_count_result['hits']['hits'][0]['_source']['total_post_num'] except Exception,e: pass
def show_qq_xnr(MAX_VALUE): query_body = {'query': {'match_all': {}}, 'size': MAX_VALUE} results = [] result = es_xnr.search(index=qq_xnr_index_name, doc_type=qq_xnr_index_type, body=query_body)['hits']['hits'] qq_xnr_list = [] for item in result: item_dict = dict() temp = item['_source'].copy() item_dict = dict(item_dict, **temp) #step1: identify login status port = item['_source']['qqbot_port'] qqnum = item['_source']['qq_number'] xnr_user_no = item['_source']['xnr_user_no'] group_dict = getgroup_v2(xnr_user_no) print 'group_dict:::', group_dict if group_dict: login_status = True else: login_status = False item_dict['login_status'] = login_status now_date = ts2datetime(time.time() - 24 * 3600) #histroy_id = item['_source']['xnr_user_no'] + '_' + now_date qq_xnr_history_count_index_name = qq_xnr_history_count_index_name_pre + now_date try: history_result = es_xnr.get(index=qq_xnr_history_count_index_name,\ doc_type=qq_xnr_history_count_index_type, id=xnr_user_no)['_source'] total_post_num = history_result['total_post_num'] daily_post_num = history_result['daily_post_num'] except: total_post_num = 0 daily_post_num = 0 group_message_index_name = group_message_index_name_pre + ts2datetime( time.time()) query_body = { 'query': { 'bool': { 'must': [{ 'term': { 'speaker_qq_number': qqnum } }, { 'term': { 'xnr_qq_number': qqnum } }] } } } try: count_result = es_xnr.count(index=group_message_index_name, doc_type=group_message_index_type, body=query_body) if count_result['_shards']['successful'] != 0: today_count = count_result['count'] else: print 'es index rank error' today_count = 0 except: today_count = 0 item_dict['daily_post_num'] = today_count item_dict['total_post_num'] = total_post_num + today_count results.append(item_dict) return results
'bool': { 'must': [{ 'term': { 'xnr_id': xnr_puid } }, { 'wildcard': { 'text': '*' + '@' + '*' } }] } } } try: results_total_day = es_xnr.count( index=wx_group_message_index_name, doc_type=wx_group_message_index_type, body=query_body_total_day) if results_total_day['_shards']['successful'] != 0: at_num_total_day = results_total_day['count'] except Exception, e: print 'at_num_total_day Exception: ', str(e) #统计 at_dict = {} at_dict['at_day'] = {} at_dict['at_total'] = {} at_dict['at_day'][current_time] = at_num_xnr at_dict['at_total'][current_time] = at_num_total influence = (float(math.log(at_num_xnr + 1)) / (math.log(at_num_total_day + 1) + 1)) * 100 #influence = (float(math.log(at_num_xnr+1))/(math.log(at_num_total_day+1)+1))
def utils_get_safe(wxbot_id, period, startdate, enddate): start_ts, end_ts, period = dump_date(period, startdate, enddate) current_timestamp = int(time.time()) current_date = ts2datetime(current_timestamp) if period == 0: #获取今天的数据 current_time = datetime2ts(current_date) last_date = ts2datetime(current_time-DAY) speak_dict = {} speak_dict['speak_day'] = {} speak_dict['speak_total'] = {} xnr_puid = load_wxxnr_redis_data(wxbot_id=wxbot_id, items=['puid'])['puid'] #获取今日发言总数 query_body = { 'query':{ 'bool':{ 'must':[ {'term':{'speaker_id': xnr_puid}}, {'term':{'xnr_id':xnr_puid}} ] } } } today_index_name = wx_group_message_index_name_pre + current_date today_count_result = es_xnr.count(index=today_index_name,doc_type=wx_group_message_index_type,body=query_body) if today_count_result['_shards']['successful'] != 0: today_count = today_count_result['count'] else: print 'es index rank error' today_count = 0 #获取历史发言总数 total_query_body = { 'query':{ 'bool':{ 'must':[ {'term':{'xnr_user_no': wxbot_id}}, {'term':{'puid':xnr_puid}}, {'term':{'date_time':last_date}} ] } } } total_index_name = wx_xnr_history_count_index_name try: total_count_result = es_xnr.search(index=total_index_name,doc_type=wx_xnr_history_count_index_type,body=total_query_body) if total_count_result['_shards']['successful'] != 0: total_count = total_count_result['hits']['hits'][0]['_source']['total_post_num'] except Exception,e: print e total_count = 0 #包括今天在内的发言总数 total_count_totay = total_count + today_count #发言次数最大值 query_body_total_day = { 'query':{ 'filtered':{ 'filter':{ 'term':{'xnr_id':xnr_puid} } } }, 'aggs':{ 'all_speakers':{ 'terms':{'field':'speaker_id',"order" : { "_count" : "desc" }} } } } try: results_total_day = es_xnr.search(index=wx_group_message_index_name,doc_type=wx_group_message_index_type,\ body=query_body_total_day)['aggregations']['all_speakers']['buckets'] speaker_max = results_total_day[0]['doc_count'] except: speaker_max = today_count #整合 speak_dict = dict() speak_dict['speak_today'] = {} speak_dict['speak_total'] = {} speak_dict['speak_today'][current_time] = today_count speak_dict['speak_total'][current_time] = total_count_totay safe_active = (float(math.log(today_count+1))/(math.log(speaker_max+1)+1))*100 safe_active = round(safe_active,2) # 保留两位小数 speak_dict['mark'] = safe_active return speak_dict
def get_influence_at_num_today(xnr_user_no): at_dict = {} at_dict['at_day'] = {} at_dict['at_total'] = {} get_result = es_xnr.get(index=qq_xnr_index_name,doc_type=qq_xnr_index_type,id=xnr_user_no)['_source'] qq_number = get_result['qq_number'] nickname = get_result['nickname'] if S_TYPE == 'test': current_time = datetime2ts(QQ_S_DATE_ASSESSMENT) else: current_time = int(time.time()) current_date = ts2datetime(current_time) group_message_index_name = group_message_index_name_pre + current_date #虚拟人今天被@数量 query_body_xnr = { 'query':{ 'bool':{ 'must':[ {'term':{'xnr_qq_number':qq_number}}, {'wildcard':{'text':'*'+'@ME'+'*'}} ] } } } try: results_xnr = es_xnr.count(index=group_message_index_name,doc_type=group_message_index_type,\ body=query_body_xnr) if results_xnr['_shards']['successful'] != 0: at_num_xnr = results_xnr['count'] else: print 'es index rank error' at_num_xnr = 0 except: at_num_xnr = 0 # 得到历史总数 current_time_last = current_time - DAY current_date_last = ts2datetime(current_time_last) qq_xnr_history_count_index_name = qq_xnr_history_count_index_name_pre + current_date_last try: result_last = es_xnr.get(index=qq_xnr_history_count_index_name,doc_type=qq_xnr_history_be_at_index_type,id=xnr_user_no)['_source'] total_be_at_num_last = result_last['total_be_at_num'] except: total_be_at_num_last = 0 at_dict['at_day'][current_time] = at_num_xnr at_dict['at_total'][current_time]= at_num_xnr + total_be_at_num_last query_body_total_day = { 'query':{ 'bool':{ 'must':[ {'term':{'xnr_qq_number':qq_number}}, {'wildcard':{'text':'*'+'@'+'*'}} ] } } } try: results_total_day = es_xnr.count(index=group_message_index_name,doc_type=group_message_index_type,\ body=query_body_total_day) if results_total_day['_shards']['successful'] != 0: at_num_total_day = results_total_day['count'] else: print 'es index rank error' at_num_total_day = 0 except: at_num_total_day = 0 influence = (float(math.log(at_num_xnr+1))/(math.log(at_num_total_day+1)+1))*100 influence = round(influence,2) # 保留两位小数 at_dict['mark'] = influence return at_dict
def get_safe_qq_today(xnr_user_no): get_result = es_xnr.get(index=qq_xnr_index_name,doc_type=qq_xnr_index_type,id=xnr_user_no)['_source'] qq_number = get_result['qq_number'] if S_TYPE == 'test': current_time = datetime2ts(QQ_S_DATE_ASSESSMENT) else: current_time = int(time.time()) current_date = ts2datetime(current_time) group_message_index_name = group_message_index_name_pre + current_date query_body = { 'query':{ 'bool':{ 'must':[ {'term':{'speaker_qq_number':qq_number}}, {'term':{'xnr_qq_number':qq_number}} ] } } } count_result = es_xnr.count(index=group_message_index_name,doc_type=group_message_index_type,body=query_body) if count_result['_shards']['successful'] != 0: today_count = count_result['count'] else: print 'es index rank error' today_count = 0 last_date = ts2datetime(current_time-DAY) qq_xnr_history_count_index_name = qq_xnr_history_count_index_name_pre + last_date try: get_result = es.get(index=qq_xnr_history_count_index_name,doc_type=qq_xnr_history_count_index_type,\ id=_id_last)['_source'] total_count_history = get_result['total_post_num'] except: total_count_history = 0 total_count_totay = total_count_history + today_count item_dict = dict() item_dict['speak_today'] = {} item_dict['speak_total'] = {} item_dict['speak_today'][current_time] = today_count item_dict['speak_total'][current_time] = total_count_totay query_body_total_day = { 'query':{ 'filtered':{ 'filter':{ 'term':{'xnr_qq_number':qq_number} } } }, 'aggs':{ 'all_speakers':{ 'terms':{'field':'speaker_qq_number',"order" : { "_count" : "desc" }} } } } try: results_total_day = es_xnr.search(index=group_message_index_name,doc_type=group_message_index_type,\ body=query_body_total_day)['aggregations']['all_speakers']['buckets'] speaker_max = results_total_day[0]['doc_count'] except: speaker_max = today_count safe_active = (float(math.log(today_count+1))/(math.log(speaker_max+1)+1))*100 safe_active = round(safe_active,2) # 保留两位小数 item_dict['mark'] = safe_active return item_dict
def utils_get_influence(wxbot_id, period, startdate, enddate): start_ts, end_ts, period = dump_date(period, startdate, enddate) current_timestamp = int(time.time()) current_date = ts2datetime(current_timestamp) if period == 0: #获取今天的数据 xnr_puid = load_wxxnr_redis_data(wxbot_id=wxbot_id, items=['puid'])['puid'] current_time = datetime2ts(current_date) query_at_num = { 'query':{ 'bool':{ 'must':[ {'term':{'xnr_id':xnr_puid}}, {'term':{'at_flag':1}} ] } } } #虚拟人今天被@数量 wx_group_message_index_name = wx_group_message_index_name_pre + current_date try: results_xnr = es_xnr.count(index=wx_group_message_index_name,doc_type=wx_group_message_index_type,body=query_at_num) if results_xnr['_shards']['successful'] != 0: at_num_xnr = results_xnr['count'] else: print 'es index rank error' at_num_xnr = 0 except: at_num_xnr = 0 # 截止目前所有被@总数 wx_group_message_index_list = get_wx_groupmessage_index_list(WX_GROUP_MESSAGE_START_DATE_ASSESSMENT,ts2datetime(current_time)) at_num_total = 0 for index_name in wx_group_message_index_list: r = es_xnr.count(index=wx_group_message_index_name,doc_type=wx_group_message_index_type,body=query_at_num) if r['_shards']['successful'] != 0: at_num_total += r['count'] #查询所有人被@的次数 query_body_total_day = { 'query':{ 'bool':{ 'must':[ {'term':{'xnr_id':xnr_puid}}, {'wildcard':{'text':'*'+'@'+'*'}} ] } } } try: results_total_day = es_xnr.count(index=wx_group_message_index_name,doc_type=wx_group_message_index_type,body=query_body_total_day) if results_total_day['_shards']['successful'] != 0: at_num_total_day = results_total_day['count'] else: print 'es index rank error' at_num_total_day = 0 except: at_num_total_day = 0 #统计 at_dict = {} at_dict['at_day'] = {} at_dict['at_total'] = {} at_dict['at_day'][current_time] = at_num_xnr at_dict['at_total'][current_time] = at_num_total influence = (float(math.log(at_num_xnr+1))/(math.log(at_num_total_day+1)+1))*100 influence = round(influence,2) # 保留两位小数 at_dict['mark'] = influence return at_dict else: at_dict = {} at_dict['at_day'] = {} at_dict['at_total'] = {} query_body = { 'query':{ 'filtered':{ 'filter':{ 'bool':{ 'must':[ {'term':{'xnr_user_no':wxbot_id}}, {'range':{'timestamp':{'gte':start_ts,'lte':end_ts}}} ] } } } }, 'size':MAX_SEARCH_SIZE, 'sort':{'timestamp':{'order':'asc'}} } search_results = es_xnr.search(index=wx_xnr_history_count_index_name,doc_type=wx_xnr_history_count_index_type,\ body=query_body)['hits']['hits'] #初始化 ts_list = load_timestamp_list(start_ts, end_ts) for ts in ts_list: at_dict['at_day'][ts] = 0 at_dict['at_total'][ts] = 0 at_dict['mark'] = 0 #填充数据 for result in search_results: result = result['_source'] timestamp = result['timestamp'] at_dict['at_day'][timestamp] = result['daily_be_at_num'] at_dict['at_total'][timestamp] = result['total_be_at_num'] at_dict['mark'] = result['influence'] return at_dict