def lookup_todayweibo_date_warming(keywords,today_datetime): keyword_query_list=[] for keyword in keywords: keyword_query_list.append({'wildcard':{'text':'*'+keyword.encode('utf-8')+'*'}}) flow_text_index_name=get_day_flow_text_index_list(today_datetime) query_body={ 'query':{ 'bool': { 'should':keyword_query_list # 'must':{'range':{'sensitive':{'gte':1}}} } }, 'size':MAX_WARMING_SIZE, 'sort':{'sensitive':{'order':'desc'}} } try: temp_result=es_flow_text.search(index=flow_text_index_name,doc_type=flow_text_index_type,body=query_body)['hits']['hits'] date_result=[] for item in temp_result: item['_source']['nick_name']=get_user_nickname(item['_source']['uid']) date_result.append(item['_source']) except: date_result=[] return date_result
def get_hashtag(today_datetime): weibo_flow_text_index_name=get_day_flow_text_index_list(today_datetime) # query_body_test={ # 'query':{ # 'match_all':{} # }, # 'size':999 # } # weibo_result_test=es_flow_text.search(index=weibo_flow_text_index_name,doc_type=flow_text_index_type,body=query_body_test)['hits']['hits'] # for item in weibo_result_test: # print 'item:',item['_source'] # if item['_source']['hashtag']: # print 'hashtag_mark:',item['_source']['hashtag'] # else: # pass query_body={ 'query':{ 'filtered':{ 'filter':{ 'bool':{ 'must':[ {'range':{'sensitive':{'gte':1}}} ] }} } }, 'aggs':{ 'all_hashtag':{ 'terms':{'field':'hashtag'}, 'aggs':{'sum_sensitive':{ 'sum':{'field':'sensitive'} } } } }, 'size':5 } weibo_text_exist=es_flow_text.search(index=weibo_flow_text_index_name,doc_type=flow_text_index_type,\ body=query_body)['aggregations']['all_hashtag']['buckets'] # print 'weibo_hashtag:',weibo_text_exist hashtag_list = [] for item in weibo_text_exist: event_dict=dict() if item['key']: # print item['key'] event_dict['event_name'] = item['key'] event_dict['event_count'] = item['doc_count'] event_dict['event_sensitive'] = item['sum_sensitive']['value'] hashtag_list.append(event_dict) else: pass hashtag_list.sort(key=lambda k:(k.get('event_sensitive',0),k.get('event_count',0)),reverse=True) # print 'hashtag_list:',hashtag_list return hashtag_list
def lookup_today_fullkeywords(from_ts,to_ts): query_body={ 'query':{ 'filtered':{ 'filter':{ 'bool':{ 'must':[ {'range':{'timestamp':{'gte':from_ts,'lte':to_ts}}} ] } } } }, 'aggs':{ 'keywords':{ 'terms':{ 'field':'keywords_string', 'size': 80 } } } } flow_text_index_name = flow_text_index_name_pre + ts2datetime(to_ts) try: flow_text_exist=es_flow_text.search(index=flow_text_index_name,doc_type=xnr_flow_text_index_type,\ body=query_body)['aggregations']['keywords']['buckets'] word_dict = dict() word_dict_new = dict() keywords_string = '' for item in flow_text_exist: word = item['key'] count = item['doc_count'] word_dict[word] = count keywords_string += '&' keywords_string += item['key'] k_dict = extract_keywords(keywords_string) for item_item in k_dict: keyword = item_item.word # print 'keyword::',keyword,type(keyword) if word_dict.has_key(keyword): word_dict_new[keyword] = word_dict[keyword] else: word_dict_new[keyword] = 1 # print 'count:',word_dict_new[keyword] except: word_dict_new = dict() return word_dict_new
def lookup_today_keywords(from_ts, to_ts, xnr_user_no): userslist = lookup_weiboxnr_concernedusers(xnr_user_no) query_body = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'terms': { 'uid': userslist } }, { 'range': { 'timestamp': { 'gte': from_ts, 'lte': to_ts } } }] } } } }, 'aggs': { 'keywords': { 'terms': { 'field': 'keywords_string', 'size': 50 } } } } flow_text_index_name = flow_text_index_name_pre + ts2datetime(to_ts) #print 'flow_text_index_name:', flow_text_index_name #try: #if S_TYPE == 'test': # temp_flow_text_index_name='flow_text_2016-11-19' # flow_text_exist=es_flow_text.search(index=temp_flow_text_index_name,doc_type=flow_text_index_type,\ # body=query_body)['aggregations']['keywords']['buckets'] #else: flow_text_exist=es_flow_text.search(index=flow_text_index_name,doc_type=xnr_flow_text_index_type,\ body=query_body)['aggregations']['keywords']['buckets'] #except: # flow_text_exist=[] word_dict = dict() for item in flow_text_exist: word = item['key'] count = item['doc_count'] word_dict[word] = count #print 'word_dict:', word_dict return word_dict
def lookup_weibo_date_warming(keywords, today_time): keyword_query_list = [] for keyword in keywords: #keyword_query_list=keyword+'' keyword_query_list.append( {'wildcard': { 'text': '*' + keyword.encode('utf-8') + '*' }}) if S_TYPE == 'test': test_time_gap = DAY * WARMING_DAY end_time = datetime2ts(S_DATE_BCI) start_time = end_time - test_time_gap flow_text_index_name_list = get_xnr_flow_text_index_listname( flow_text_index_name_pre, start_time, end_time) #print flow_text_index_name_list else: start_time = today_time - DAY * WARMING_DAY flow_text_index_name_list = get_xnr_flow_text_index_listname( flow_text_index_name_pre, start_time, today_time) #print start_time,end_time,flow_text_index_name_pre,flow_text_index_name_list query_body = { 'query': { 'bool': { 'should': keyword_query_list } }, 'size': SPEECH_WARMING_NUM } try: temp_result = es_flow_text.search(index=flow_text_index_name_list, doc_type=flow_text_index_type, body=query_body)['hits']['hits'] date_result = [] #print temp_result for item in temp_result: date_result.append(item['_source']) except: date_result = [] return date_result
def lookup_today_speech_warming(xnr_user_no,show_type,start_time,end_time): #查询关注列表 lookup_type='followers_list' followers_list=lookup_xnr_fans_followers(xnr_user_no,lookup_type) show_condition_list=[] if show_type == 0: #全部用户 show_condition_list.append({'must':[{'range':{'sensitive':{'gte':1}}},{'range':{'timestamp':{'gte':start_time,'lte':end_time}}}]}) elif show_type == 1: #关注用户 show_condition_list.append({'must':[{'terms':{'uid':followers_list}},{'range':{'sensitive':{'gte':1}}},{'range':{'timestamp':{'gte':start_time,'lte':end_time}}}]}) elif show_type ==2: #未关注用户 show_condition_list.append({'must_not':{'terms':{'uid':followers_list}},'must':[{'range':{'sensitive':{'gte':1}}},{'range':{'timestamp':{'gte':start_time,'lte':end_time}}}]}) query_body={ 'query':{ 'filtered':{ 'filter':{ 'bool':show_condition_list[0] } } }, 'size':SPEECH_WARMING_NUM, 'sort':{'sensitive':{'order':'desc'}} } flow_text_index_name=get_day_flow_text_index_list(start_time) result=[] try: results=es_flow_text.search(index=flow_text_index_name,doc_type=flow_text_index_type,body=query_body)['hits']['hits'] for item in results: item['_source']['nick_name']=get_user_nickname(item['_source']['uid']) item['_source']['_id']=item['_id'] result.append(item['_source']) except: result=[] return result
def get_generate_example_model(domain_name,role_name): domain_pinyin = pinyin.get(domain_name,format='strip',delimiter='_') role_en = domain_ch2en_dict[role_name] task_id = domain_pinyin + '_' + role_en es_result = es.get(index=weibo_role_index_name,doc_type=weibo_role_index_type,id=task_id)['_source'] item = es_result print 'es_result:::',es_result # 政治倾向 political_side = json.loads(item['political_side'])[0][0] if political_side == 'mid': item['political_side'] = u'中立' elif political_side == 'left': item['political_side'] = u'左倾' else: item['political_side'] = u'右倾' # 心理特征 psy_feature_list = [] psy_feature = json.loads(item['psy_feature']) for i in range(TOP_PSY_FEATURE): psy_feature_list.append(psy_feature[i][0]) item['psy_feature'] = '&'.join(psy_feature_list) role_group_uids = json.loads(item['member_uids']) mget_results = es_user_portrait.mget(index=portrait_index_name,doc_type=portrait_index_type,body={'ids':role_group_uids})['docs'] # topic_list = [] # for mget_item in mget_results: # if mget_item['found']: # keywords_list = json.loads(mget_item['_source']['keywords']) # topic_list.extend(keywords_list) # topic_keywords_dict = {} # for topic_item in topic_list: # keyword = topic_item[0] # keyword_count = topic_item[1] # try: # topic_keywords_dict[keyword] += keyword_count # except: # topic_keywords_dict[keyword] = keyword_count # monitor_keywords_list = [] # for i in range(3): # keyword_max = max(topic_keywords_dict,key=topic_keywords_dict.get) # monitor_keywords_list.append(keyword_max) # del topic_keywords_dict[keyword_max] # item['monitor_keywords'] = '&'.join(monitor_keywords_list) if S_TYPE == 'test': current_time = datetime2ts(S_DATE) else: current_time = int(time.time()) index_name_list = get_flow_text_index_list(current_time) query_body_search = { 'query':{ 'filtered':{ 'filter':{ 'terms':{'uid':role_group_uids} } } }, 'size':MAX_VALUE, '_source':['keywords_string'] } es_keyword_results = es_flow_text.search(index=index_name_list,doc_type=flow_text_index_type,\ body=query_body_search)['hits']['hits'] keywords_string = '' for mget_item in es_keyword_results: #print 'mget_item:::',mget_item #if mget_item['found']: keywords_string += '&' keywords_string += mget_item['_source']['keywords_string'] k_dict = extract_keywords(keywords_string) monitor_keywords_list = [] for item_item in k_dict: monitor_keywords_list.append(item_item.word.encode('utf-8')) item['monitor_keywords'] = ','.join(monitor_keywords_list) mget_results_user = es_user_portrait.mget(index=profile_index_name,doc_type=profile_index_type,body={'ids':role_group_uids})['docs'] item['nick_name'] = [] for mget_item in mget_results_user: #print 'mget_item:::',mget_item if mget_item['found']: item['nick_name'] = mget_item['_source']['nick_name'] item['location'] = mget_item['_source']['user_location'] item['gender'] = mget_item['_source']['sex'] uid = mget_item['_source']['uid'] try: profile_results = es_user_portrait.get(index=profile_index_name,doc_type=profile_index_type,id=uid)['_source'] if profile_results['description']: item['description'] = profile_results['description'] break except: pass item['business_goal'] = u'渗透' item['daily_interests'] = u'旅游' # if S_TYPE == 'test': # user_mget_results = es.mget(index=profile_index_name,doc_type=profile_index_type,body={'ids':role_group_uids})['docs'] # if user_mget_results item['age'] = 30 item['career'] = u'自由职业' active_time_list_np = np.array(json.loads(item['active_time'])) active_time_list_np_sort = np.argsort(-active_time_list_np)[:TOP_ACTIVE_TIME] item['active_time'] = active_time_list_np_sort.tolist() day_post_num_list = np.array(json.loads(item['day_post_num'])) item['day_post_num'] = np.mean(day_post_num_list).tolist() item['role_name'] = role_name task_id_new =domain_pinyin + '_' + role_en example_model_file_name = EXAMPLE_MODEL_PATH + task_id_new + '.json' try: with open(example_model_file_name,"w") as dump_f: json.dump(item,dump_f) item_dict = dict() #item_dict['xnr_user_no'] = xnr_user_no item_dict['domain_name'] = domain_name item_dict['role_name'] = role_name es.index(index=weibo_example_model_index_name,doc_type=weibo_example_model_index_type,\ body=item_dict,id=task_id_new) mark = True except: mark = False return mark
def get_community_content(now_time, uid_list, order_by): #step1:获取uid_list #step2:生成话题词云、敏感词云、敏感帖子 query_body = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'terms': { 'uid': uid_list } }] } } } }, 'aggs': { 'sensitive_words_string': { 'terms': { 'field': 'sensitive_words_string', 'size': 800 } } }, 'size': 50, 'sort': { order_by: { 'order': 'desc' } } } query_hashtag = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'terms': { 'uid': uid_list } }] } } } }, 'aggs': { 'hashtag': { 'terms': { 'field': 'hashtag', 'size': 800 } } }, 'size': 50, 'sort': { order_by: { 'order': 'desc' } } } flow_text_index_name_list = get_flow_text_index_list(now_time) flow_text_exist = es_flow_text.search(index = flow_text_index_name_list,doc_type = flow_text_index_type,\ body = query_body) sensitive_keywords_result = flow_text_exist['aggregations'][ 'sensitive_words_string']['buckets'] content_result = flow_text_exist['hits']['hits'] hashtag_result = es_flow_text.search(index = flow_text_index_name_list,doc_type = flow_text_index_type,\ body = query_hashtag)['aggregations']['hashtag']['buckets'] # print 'content_result:',content_result # print 'sensitive_keywords_result:',sensitive_keywords_result # print 'hashtag_result:',hashtag_result sensitive_keywords_dict = get_word_count(sensitive_keywords_result) hashtag_dict = get_word_count(hashtag_result) content_list = [] for item in content_result: content_list.append(item['_source']) result_dict = dict() result_dict['topic_wordcloud'] = hashtag_dict result_dict['sensitive_wordcloud'] = sensitive_keywords_dict result_dict['content_post'] = content_list return result_dict
def create_event_warning(xnr_user_no,today_datetime,write_mark): #获取事件名称 hashtag_list = get_hashtag(today_datetime) #print 'hashtag_list::',hashtag_list flow_text_index_name = get_day_flow_text_index_list(today_datetime) #虚拟人的粉丝列表和关注列表 try: es_xnr_result=es_xnr.get(index=weibo_xnr_fans_followers_index_name,doc_type=weibo_xnr_fans_followers_index_type,id=xnr_user_no)['_source'] followers_list=es_xnr_result['followers_list'] fans_list=es_xnr_result['fans_list'] except: followers_list=[] fans_list=[] event_warming_list=[] event_num=0 for event_item in hashtag_list: event_sensitive_count=0 event_warming_content=dict() #事件名称、主要参与用户、典型微博、事件影响力、事件平均时间 event_warming_content['event_name']=event_item['event_name'] # print 'event_name:',event_item event_num=event_num+1 # print 'event_num:::',event_num # print 'first_time:::',int(time.time()) event_influence_sum=0 event_time_sum=0 query_body={ 'query':{ # 'bool':{ # 'must':[{'wildcard':{'text':'*'+event_item[0]+'*'}}, # {'range':{'sensitive':{'gte':1}}}] # } 'filtered':{ 'filter':{ 'bool':{ 'must':[ {'term':{'hashtag':event_item['event_name']}}, {'range':{'sensitive':{'gte':1}}} ] } } } }, 'size':MAX_WARMING_SIZE, 'sort':{'sensitive':{'order':'desc'}} } #try: event_results=es_flow_text.search(index=flow_text_index_name,doc_type=flow_text_index_type,body=query_body)['hits']['hits'] if event_results: weibo_result=[] fans_num_dict=dict() followers_num_dict=dict() alluser_num_dict=dict() # print 'sencond_time:::',int(time.time()) for item in event_results: #print 'event_content:',item['_source']['text'] #统计用户信息 if alluser_num_dict.has_key(str(item['_source']['uid'])): followers_mark=set_intersection(item['_source']['uid'],followers_list) if followers_mark > 0: alluser_num_dict[str(item['_source']['uid'])]=alluser_num_dict[str(item['_source']['uid'])]+1*2 else: alluser_num_dict[str(item['_source']['uid'])]=alluser_num_dict[str(item['_source']['uid'])]+1 else: alluser_num_dict[str(item['_source']['uid'])]=1 #计算影响力 origin_influence_value=(1+item['_source']['comment']+item['_source']['retweeted'])*(1+item['_source']['sensitive']) # fans_value=judge_user_type(item['_source']['uid'],fans_list) followers_value=judge_user_type(item['_source']['uid'],followers_list) item['_source']['weibo_influence_value']=origin_influence_value*(followers_value) item['_source']['nick_name']=get_user_nickname(item['_source']['uid']) weibo_result.append(item['_source']) #统计影响力、时间 event_influence_sum=event_influence_sum+item['_source']['weibo_influence_value'] event_time_sum=event_time_sum+item['_source']['timestamp'] # print 'third_time:::',int(time.time()) #典型微博信息 weibo_result.sort(key=lambda k:(k.get('weibo_influence_value',0)),reverse=True) event_warming_content['main_weibo_info']=json.dumps(weibo_result) #事件影响力和事件时间 number=len(event_results) event_warming_content['event_influence']=event_influence_sum/number event_warming_content['event_time']=event_time_sum/number # except: # event_warming_content['main_weibo_info']=[] # event_warming_content['event_influence']=0 # event_warming_content['event_time']=0 # try: #对用户进行排序 alluser_num_dict=sorted(alluser_num_dict.items(),key=lambda d:d[1],reverse=True) main_userid_list=[] for i in xrange(0,len(alluser_num_dict)): main_userid_list.append(alluser_num_dict[i][0]) #主要参与用户信息 main_user_info=[] user_es_result=es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,body={'ids':main_userid_list})['docs'] for item in user_es_result: user_dict=dict() if item['found']: user_dict['photo_url']=item['_source']['photo_url'] user_dict['uid']=item['_id'] user_dict['nick_name']=item['_source']['nick_name'] user_dict['favoritesnum']=item['_source']['favoritesnum'] user_dict['fansnum']=item['_source']['fansnum'] else: user_dict['photo_url']='' user_dict['uid']=item['_id'] user_dict['nick_name']='' user_dict['favoritesnum']=0 user_dict['fansnum']=0 main_user_info.append(user_dict) event_warming_content['main_user_info']=json.dumps(main_user_info) # except: # event_warming_content['main_user_info']=[] # print 'fourth_time:::',int(time.time()) event_warming_content['xnr_user_no']=xnr_user_no event_warming_content['validity']=0 event_warming_content['timestamp']=today_datetime now_time=int(time.time()) # task_id=xnr_user_no+'_'+str(now_time) event_warming_content['_id']=xnr_user_no+'_'+event_warming_content['event_name'] task_id=event_warming_content['_id'] if write_mark: # print 'today_datetime:::',ts2datetime(today_datetime) mark=write_envent_warming(today_datetime,event_warming_content,task_id) event_warming_list.append(mark) else: event_warming_list.append(event_warming_content) else: pass # print 'fifth_time:::',int(time.time()) return event_warming_list
def lookup_today_personal_warming(xnr_user_no,start_time,end_time): #查询关注列表 lookup_type='followers_list' followers_list=lookup_xnr_fans_followers(xnr_user_no,lookup_type) #查询虚拟人uid xnr_uid=lookup_xnr_uid(xnr_user_no) #计算敏感度排名靠前的用户 query_body={ 'query':{ 'filtered':{ 'filter':{ 'bool':{ 'must':[ # {'terms':{'uid':followers_list}}, {'range':{ 'timestamp':{ 'gte':start_time, 'lte':end_time } }} ] } } } }, 'aggs':{ 'followers_sensitive_num':{ 'terms':{'field':'uid'}, 'aggs':{ 'sensitive_num':{ 'sum':{'field':'sensitive'} } } } }, 'size':MAX_SEARCH_SIZE } flow_text_index_name=get_day_flow_text_index_list(end_time) try: first_sum_result=es_flow_text.search(index=flow_text_index_name,doc_type=flow_text_index_type,\ body=query_body)['aggregations']['followers_sensitive_num']['buckets'] except: first_sum_result=[] #print first_sum_result top_userlist=[] for i in xrange(0,len(first_sum_result)): user_sensitive=first_sum_result[i]['sensitive_num']['value'] if user_sensitive > 0: user_dict=dict() user_dict['uid']=first_sum_result[i]['key'] user_dict['sensitive']=user_sensitive top_userlist.append(user_dict) else: pass #查询敏感用户的敏感微博内容 results=[] for user in top_userlist: #print user user_detail=dict() user_detail['uid']=user['uid'] user_detail['user_sensitive']=user['sensitive'] user_lookup_id=xnr_uid+'_'+user['uid'] print user_lookup_id try: #user_result=es_xnr.get(index=weibo_feedback_follow_index_name,doc_type=weibo_feedback_follow_index_type,id=user_lookup_id)['_source'] user_result=es_user_profile.get(index=profile_index_name,doc_type=profile_index_type,id=user['uid'])['_source'] user_detail['user_name']=user_result['nick_name'] except: user_detail['user_name']='' query_body={ 'query':{ 'filtered':{ 'filter':{ 'bool':{ 'must':[ {'term':{'uid':user['uid']}}, {'range':{'sensitive':{'gte':1}}} ] } } } }, 'size':MAX_WARMING_SIZE, 'sort':{'sensitive':{'order':'desc'}} } try: second_result=es_flow_text.search(index=flow_text_index_name,doc_type=flow_text_index_type,body=query_body)['hits']['hits'] except: second_result=[] s_result=[] for item in second_result: item['_source']['nick_name']=get_user_nickname(item['_source']['uid']) s_result.append(item['_source']) s_result.sort(key=lambda k:(k.get('sensitive',0)),reverse=True) user_detail['content']=json.dumps(s_result) user_detail['xnr_user_no']=xnr_user_no user_detail['validity']=0 user_detail['timestamp']=end_time user_detail['_id']=xnr_user_no+'_'+user_detail['uid'] results.append(user_detail) results.sort(key=lambda k:(k.get('user_sensitive',0)),reverse=True) return results
def get_recommend_follows(task_detail): recommend_results = dict() daily_interests_list = task_detail['daily_interests'].encode('utf-8').split(',') monitor_keywords_list = task_detail['monitor_keywords'].encode('utf-8').split(',') #print 'daily_interests_list::',daily_interests_list create_time = time.time() if S_TYPE == 'test': create_time = datetime2ts(S_DATE) index_name_list = get_flow_text_index_list(create_time) ## 日常兴趣关注 try: query_body = { 'query':{ 'filtered':{ 'filter':{ 'terms':{'daily_interests':daily_interests_list} } } }, 'sort':{'user_fansnum':{'order':'desc'}}, 'size':DAILY_INTEREST_TOP_USER, '_source':['uid'] } es_results = es_flow_text.search(index=index_name_list,doc_type='text',body=query_body)['hits']['hits'] daily_interest_uid_set = set() for result in es_results: daily_interest_uid_set.add(result['_source']['uid']) daily_interest_uid_list = list(daily_interest_uid_set) es_daily_interests_results = es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,\ body={'ids':daily_interest_uid_list})['docs'] nick_name_dict = {} es_daily_interests_results = es_daily_interests_results[:max(NICK_NAME_TOP,len(es_daily_interests_results))] for result in es_daily_interests_results: if result['found'] == True: result = result['_source'] nick_name_dict[result['uid']] = result['nick_name'] else: continue recommend_results['daily_interests'] = nick_name_dict except: print '没有找到日常兴趣相符的用户' recommend_results['daily_interests'] = {} ## 监测词关注 nest_query_list = [] #print 'monitor_keywords_list:::',monitor_keywords_list for monitor_keyword in monitor_keywords_list: nest_query_list.append({'wildcard':{'keywords_string':'*'+monitor_keyword+'*'}}) #print 'nest_query_list::',nest_query_list try: query_body_monitor = { 'query':{ 'bool':{ 'must':nest_query_list } }, 'sort':{'user_fansnum':{'order':'desc'}}, 'size':MONITOR_TOP_USER, '_source':['uid'] } #print '123' es_results = es_flow_text.search(index=index_name_list,doc_type='text',body=query_body_monitor)['hits']['hits'] #print 'es_results::',es_results monitor_keywords_uid_set = set() for result in es_results: monitor_keywords_uid_set.add(result['_source']['uid']) monitor_keywords_uid_list = list(monitor_keywords_uid_set) es_monitor_keywords_results = es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,\ body={'ids':monitor_keywords_uid_list})['docs'] nick_name_dict = {} es_monitor_keywords_results = es_monitor_keywords_results[:max(NICK_NAME_TOP,len(es_monitor_keywords_results))] for result in es_monitor_keywords_results: if result['found'] == True: result = result['_source'] nick_name_dict[result['uid']] = result['nick_name'] else: continue recommend_results['monitor_keywords'] = nick_name_dict except: print '没有找到监测词相符的用户' recommend_results['monitor_keywords'] = {} # print 'recommend_results::',recommend_results return recommend_results
def lookup_hot_posts(from_ts, to_ts, weiboxnr_id, classify_id, order_id): #step 1 :adjust the time condition for time time_gap = to_ts - from_ts now_time = time.time() test_time_gap = datetime2ts(ts2datetime(now_time)) - datetime2ts(S_DATE) #print 'from, to:', from_ts, to_ts if S_TYPE == 'test': today_date_time = datetime2ts(S_DATE) from_ts = from_ts - test_time_gap #to_ts = to_ts - test_time_gap to_ts = from_ts + MAX_FLOW_TEXT_DAYS * DAY from_date_ts = datetime2ts(ts2datetime(from_ts)) to_date_ts = datetime2ts(ts2datetime(to_ts)) #print 'from_date_ts, to_date_ts:', ts2date(from_date_ts), ts2date(to_date_ts) #print from_date_ts,to_date_ts flow_text_index_name_list = [] days_num = MAX_FLOW_TEXT_DAYS for i in range(0, (days_num + 1)): date_range_start_ts = to_date_ts - i * DAY date_range_start_datetime = ts2datetime(date_range_start_ts) index_name = flow_text_index_name_pre + date_range_start_datetime if es_flow_text.indices.exists(index=index_name): flow_text_index_name_list.append(index_name) else: pass if order_id == 1: #按时间排序 sort_condition_list = [{'timestamp': {'order': 'desc'}}] elif order_id == 2: #按热度排序 sort_condition_list = [{'retweeted': {'order': 'desc'}}] elif order_id == 3: #按敏感度排序 sort_condition_list = [{'sensitive': {'order': 'desc'}}] #else: #默认设为按时间排序 # sort_condition_list=[{'timestamp':{'order':'desc'}}] userslist = lookup_weiboxnr_concernedusers(weiboxnr_id) #全部用户 0,已关注用户 1,未关注用户-1 range_time_list = { 'range': { 'timestamp': { 'gte': int(from_ts), 'lt': int(to_ts) } } } print range_time_list user_condition_list = [] if classify_id == 1: user_condition_list = [{ 'bool': { 'must': [{ 'terms': { 'uid': userslist } }, range_time_list] } }] elif classify_id == 2: user_condition_list = [{ 'bool': { 'must': [range_time_list], 'must_not': [{ 'terms': { 'uid': userslist } }] } }] elif classify_id == 0: user_condition_list = [{'bool': {'must': [range_time_list]}}] #print 'sort_condition_list',sort_condition_list #print 'user_condition_list',user_condition_list query_body = { 'query': { 'filtered': { 'filter': user_condition_list } }, 'size': HOT_WEIBO_NUM, 'sort': sort_condition_list } try: es_result=es_flow_text.search(index=flow_text_index_name_list,doc_type=flow_text_index_type,\ body=query_body)['hits']['hits'] hot_result = [] for item in es_result: item['_source']['nick_name'] = get_user_nickname( item['_source']['uid']) hot_result.append(item['_source']) except: hot_result = [] #print 'hot_result:', hot_result return hot_result
def show_event_warming(xnr_user_no): now_time = int(time.time()) #print 'first_time:',time.time() hashtag_list = get_hashtag() #print 'hashtag_list_time:',time.time() #print 'hashtag_list:::::::',hashtag_list if S_TYPE == 'test': test_day_date = S_DATE_EVENT_WARMING test_day_time = datetime2ts(test_day_date) flow_text_index_list = get_flow_text_index_list(test_day_time) #print flow_text_index_list hashtag_list = [('网络义勇军发布', 13), ('美国', 7), ('德国', 5), ('中国', 4), ('清真食品', 3), ('反邪动态', 2), ('台海观察', 2), ('雷哥微评', 2), ('中国军队', 1)] #hashtag_list=[('网络义勇军发布',13),('美国',7),('芒果TV',6),('德国',5),('中国',4),('清真食品',3),('反邪动态',2),('台海观察',2),('每日一药',2),('雷哥微评',2),('PPAP洗脑神曲',1),('中国军队',1)] #weibo_xnr_flow_text_listname=['flow_text_2016-11-26','flow_text_2016-11-25','flow_text_2016-11-24'] else: flow_text_index_list = get_flow_text_index_list(now_time) #weibo_xnr_flow_text_listname=get_xnr_flow_text_index_list(now_time) #print flow_text_index_list,hashtag_list #虚拟人的粉丝列表和关注列表 try: es_xnr_result = es_xnr.get( index=weibo_xnr_fans_followers_index_name, doc_type=weibo_xnr_fans_followers_index_type, id=xnr_user_no)['_source'] followers_list = es_xnr_result['followers_list'] fans_list = es_xnr_result['fans_list'] except: followers_list = [] fans_list = [] #print 'weibo_xnr_fans_followers_time:',time.time() event_warming_list = [] for event_item in hashtag_list: #print event_item,event_item[0] event_sensitive_count = 0 event_warming_content = dict() #事件名称、主要参与用户、典型微博、事件影响力、事件平均时间 event_warming_content['event_name'] = event_item[0] event_influence_sum = 0 event_time_sum = 0 query_body = { 'query': { 'bool': { 'should': { 'wildcard': { 'text': '*' + event_item[0] + '*' } } } } } try: event_results = es_flow_text.search( index=flow_text_index_list, doc_type=flow_text_index_type, body=query_body)['hits']['hits'] weibo_result = [] fans_num_dict = dict() followers_num_dict = dict() alluser_num_dict = dict() #print event_results for item in event_results: if item['_source']['sensitive'] > 0: event_sensitive_count = event_sensitive_count + 1 #统计用户信息 if alluser_num_dict.has_key(str(item['_source']['uid'])): alluser_num_dict[str( item['_source']['uid'])] = alluser_num_dict[str( item['_source']['uid'])] + 1 else: alluser_num_dict[str(item['_source']['uid'])] = 1 for fans_uid in fans_list: if fans_uid == item['_source']['uid']: if fans_num_dict.has_key(str(fans_uid)): fans_num_dict[str(fans_uid)] = fans_num_dict[ str(fans_uid)] + 1 else: fans_num_dict[str(fans_uid)] = 1 for followers_uid in followers_list: if followers_uid == item['_source']['uid']: if followers_num_dict.has_key(str(followers_uid)): fans_num_dict[str( followers_uid )] = fans_num_dict[str(followers_uid)] + 1 else: fans_num_dict[str(followers_uid)] = 1 #计算影响力 origin_influence_value = (item['_source']['comment'] + item['_source']['retweeted']) * ( 1 + item['_source']['sensitive']) fans_value = judge_user_type(item['_source']['uid'], fans_list) followers_value = judge_user_type(item['_source']['uid'], followers_list) item['_source'][ 'weibo_influence_value'] = origin_influence_value * ( fans_value + followers_value) weibo_result.append(item['_source']) #统计影响力、时间 event_influence_sum = event_influence_sum + item[ '_source']['weibo_influence_value'] event_time_sum = item['_source']['timestamp'] #典型微博信息 weibo_result.sort(key=lambda k: (k.get('weibo_influence_value', 0)), reverse=True) event_warming_content['main_weibo_info'] = weibo_result #事件影响力和事件时间 number = len(event_results) event_warming_content[ 'event_influence'] = event_influence_sum / number event_warming_content['event_time'] = event_time_sum / number else: pass except: event_warming_content['main_weibo_info'] = [] event_warming_content['event_influence'] = [] event_warming_content['event_time'] = [] #print event_item[0],'event_search_time:',time.time() try: if event_sensitive_count > 0: #对用户进行排序 temp_userid_dict = union_dict(fans_num_dict, followers_num_dict) main_userid_dict = union_dict(temp_userid_dict, alluser_num_dict) main_userid_dict = sorted(main_userid_dict.items(), key=lambda d: d[1], reverse=True) main_userid_list = [] for i in xrange(0, len(main_userid_dict)): main_userid_list.append(main_userid_dict[i][0]) #print 'main_userid_list:',main_userid_list #主要参与用户信息 main_user_info = [] user_es_result = es_user_profile.mget( index=profile_index_name, doc_type=profile_index_type, body={'ids': main_userid_list})['docs'] for item in user_es_result: #print 'item:',item #print 'found:',item['found'] #print 'id:',item['_id'] user_dict = dict() if item['found']: user_dict['photo_url'] = item['_source']['photo_url'] user_dict['uid'] = item['_id'] user_dict['nick_name'] = item['_source']['nick_name'] user_dict['favoritesnum'] = item['_source'][ 'favoritesnum'] user_dict['fansnum'] = item['_source']['fansnum'] else: user_dict['photo_url'] = '' user_dict['uid'] = item['_id'] user_dict['nick_name'] = '' user_dict['favoritesnum'] = 0 user_dict['fansnum'] = 0 main_user_info.append(user_dict) event_warming_content['main_user_info'] = main_user_info #print 'main_user_info:',main_user_info #print user_es_result ''' user_query_body={ 'query':{ 'filtered':{ 'filter':{ 'terms':{'uid':main_userid_list} } } } } user_es_result=es_user_profile.search(index=profile_index_name,doc_type=profile_index_type,body=user_query_body)['hits']['hits'] #print user_es_result main_user_info=[] for item in user_es_result: main_user_info.append(item['_source']) event_warming_content['main_user_info']=main_user_info ''' else: event_warming_content['main_user_info'] = [] except: event_warming_content['main_user_info'] = [] #print 'user_search_time:',time.time() if event_sensitive_count > 0: #print event_warming_content['event_name'] event_warming_list.append(event_warming_content) else: pass #main_userid_list=['5536381570','2192435767','1070598590'] #user_es_result=es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,body={'ids':main_userid_list}) #print 'user_es_result',user_es_result #print 'end_time:',time.time() return event_warming_list
def show_personnal_warming(xnr_user_no, day_time): #查询关注列表 try: es_xnr_result = es_xnr.get( index=weibo_xnr_fans_followers_index_name, doc_type=weibo_xnr_fans_followers_index_type, id=xnr_user_no)['_source'] followers_list = es_xnr_result['followers_list'] except: followers_list = [] #计算敏感度排名靠前的用户 query_body = { 'query': { 'filtered': { 'filter': { 'terms': { 'uid': followers_list } } } }, 'aggs': { 'followers_sensitive_num': { 'terms': { 'field': 'uid' }, 'aggs': { 'sensitive_num': { 'sum': { 'field': 'sensitive' } } } } }, 'size': MAX_VALUE } #测试状态下时间设置 if S_TYPE == 'test': test_day_date = S_DATE_BCI test_day_time = datetime2ts(test_day_date) flow_text_index_list = get_flow_text_index_list(test_day_time) else: flow_text_index_list = get_flow_text_index_list(day_time) #print flow_text_index_list try: first_sum_result=es_flow_text.search(index=flow_text_index_list,doc_type=flow_text_index_type,\ body=query_body)['aggregations']['followers_sensitive_num']['buckets'] except: first_sum_result = [] #print first_sum_result top_userlist = [] if USER_NUM < len(first_sum_result): temp_num = USER_NUM else: temp_num = len(first_sum_result) #print temp_num for i in xrange(0, temp_num): user_sensitive = first_sum_result[i]['sensitive_num']['value'] if user_sensitive > 0: user_dict = dict() user_dict['uid'] = first_sum_result[i]['key'] user_dict['sensitive'] = user_sensitive top_userlist.append(user_dict) else: pass #查询敏感用户的最敏感微博内容 results = [] for user in top_userlist: #print user user_detail = dict() user_detail['uid'] = user['uid'] user_detail['user_sensitive'] = user['sensitive'] try: user_result = es_user_profile.get(index=profile_index_name, doc_type=profile_index_type, id=user['uid'])['_source'] user_detail['user_name'] = user_result['nick_name'] except: user_detail['user_name'] = '' query_body = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'term': { 'uid': user['uid'] } }, { 'range': { 'sensitive': { 'gte': 1, 'lte': 100 } } }] } } } }, 'size': USER_CONTENT_NUM, 'sort': { 'sensitive': { 'order': 'desc' } } } #if S_TYPE == 'test': try: second_result = es_flow_text.search( index=flow_text_index_list, doc_type=flow_text_index_type, body=query_body)['hits']['hits'] except: second_result = [] #else: # second_result=es_xnr.search(index=weibo_xnr_flow_text_listname,doc_type=xnr_flow_text_index_type,body=query_body)['hits']['hits'] s_result = [] tem_word_one = '静坐' tem_word_two = '集合' for item in second_result: sensitive_words = item['_source']['sensitive_words_string'] if ((sensitive_words == tem_word_one) or (sensitive_words == tem_word_two)): pass else: s_result.append(item['_source']) s_result.sort(key=lambda k: (k.get('sensitive', 0)), reverse=True) user_detail['content'] = s_result results.append(user_detail) results.sort(key=lambda k: (k.get('user_sensitive', 0)), reverse=True) return results
def show_speech_warming(xnr_user_no, show_type, day_time): #关注用户 try: es_xnr_result = es_xnr.get( index=weibo_xnr_fans_followers_index_name, doc_type=weibo_xnr_fans_followers_index_type, id=xnr_user_no)['_source'] followers_list = es_xnr_result['followers_list'] except: followers_list = [] show_condition_list = [] if show_type == 0: #全部用户 show_condition_list.append( {'must': { 'range': { 'sensitive': { 'gte': 1, 'lte': 100 } } }}) elif show_type == 1: #关注用户 show_condition_list.append({ 'must': [{ 'terms': { 'uid': followers_list } }, { 'range': { 'sensitive': { 'gte': 1, 'lte': 100 } } }] }) elif show_type == 2: #未关注用户 show_condition_list.append({ 'must_not': { 'terms': { 'uid': followers_list } }, 'must': { 'range': { 'sensitive': { 'gte': 1, 'lte': 100 } } } }) query_body = { 'query': { 'filtered': { 'filter': { 'bool': show_condition_list[0] } } }, 'size': SPEECH_WARMING_NUM, 'sort': { 'sensitive': { 'order': 'desc' } } } #测试状态下时间设置 if S_TYPE == 'test': test_day_date = S_DATE_BCI test_day_time = datetime2ts(test_day_date) flow_text_index_list = get_flow_text_index_list(test_day_time) else: flow_text_index_list = get_flow_text_index_list(day_time) #try: results = es_flow_text.search(index=flow_text_index_list, doc_type=flow_text_index_type, body=query_body)['hits']['hits'] result = [] un_id_list = [ '4045093692450438', '4045096116622444', '4045095374193153', '4045095567336676', '4045092304116237', '4045093297982719', '4045178576337277', '4044647661388452' ] for item in results: if item['_id'] in un_id_list: pass else: result.append(item['_source']) #except: # result=[] return result
def lookup_active_weibouser(classify_id,weiboxnr_id,start_time,end_time): time_gap = end_time - start_time now_time = time.time() test_time_gap = datetime2ts(ts2datetime(now_time)) - datetime2ts(S_DATE_BCI) #print 'from, to:', ts2date(start_time), ts2date(end_time) if S_TYPE == 'test': today_date_time = datetime2ts(S_DATE_BCI) start_time = start_time - test_time_gap end_time = end_time - test_time_gap from_date_ts=datetime2ts(ts2datetime(start_time)) to_date_ts=datetime2ts(ts2datetime(end_time)) #print 's_date_bci:', S_DATE_BCI #print 'from_date_ts, to_date_ts:', ts2date(from_date_ts), ts2date(to_date_ts) bci_index_name = weibo_bci_index_name_pre + ''.join(ts2datetime(today_date_time).split('-')) #print 'bci_index_name:', bci_index_name #print 'end_time:', ts2date(end_time) #step1: users condition #make sure the users range by classify choice userlist = lookup_weiboxnr_concernedusers(weiboxnr_id) if classify_id == 1: #concrenedusers condition_list=[{'bool':{'must':{'terms':{'uid':userlist}}}}] elif classify_id == 2: #unconcrenedusers condition_list=[{'bool':{'must_not':[{'terms':{'uid':userlist}}]}}] elif classify_id == 0: condition_list=[{'match_all':{}}] print userlist,classify_id,condition_list #step 2:lookup users user_max_index=count_maxweibouser_influence(end_time) results = [] for item in condition_list: query_body={ 'query':item, 'size':HOT_WEIBO_NUM, #查询影响力排名前50的用户即可 'sort':{'user_index':{'order':'desc'}} } try: #print 'query_body:', query_body flow_text_exist=es_flow_text.search(index=bci_index_name,\ doc_type=weibo_bci_index_type,body=query_body)['hits']['hits'] search_uid_list = [item['_source']['user'] for item in flow_text_exist] weibo_user_exist = es_user_profile.search(index=profile_index_name,\ doc_type=profile_index_type,body={'query':{'terms':{'uid':search_uid_list}}})['hits']['hits'] #print 'weibo_user_exist:', weibo_user_exist weibo_user_dict = dict() for item in weibo_user_exist: uid = item['_source']['uid'] weibo_user_dict[uid] = item['_source'] for item in flow_text_exist: #print 'item:', item['_source'] influence = item['_source']['user_index']/user_max_index*100 fans_num = item['_source']['user_fansnum'] friends_num = item['_source']['user_friendsnum'] total_number = item['_source']['total_number'] uid = item['_source']['user'] try: weibo_user_info = weibo_user_dict[uid] uname = weibo_user_info['nick_name'] location = weibo_user_info['user_location'] url = weibo_user_info['photo_url'] except: uname = '' location = '' url = '' #print 'uid:', uid results.append({'uid':uid, 'influence':influence, 'fans_num':fans_num, \ 'total_number':total_number, 'friends_num':friends_num,\ 'uname': uname, 'location':location, 'url': url}) #print 'results:', results ''' uid=item['_source']['uid'] #微博数 item['_source']['weibos_sum']=count_weibouser_weibosum(uid,end_time) #影响力 user_index=count_weibouser_index(uid,end_time) if user_max_index >0: item['_source']['influence']=user_index/user_max_index*100 else: item['_source']['influence']=0 if item['_source']['influence']>=INFLUENCE_MIN: results.append(item['_source']) ''' except: results=[] return results