def get_generate_example_model(domain_name,role_name): domain_pinyin = pinyin.get(domain_name,format='strip',delimiter='_') role_en = domain_ch2en_dict[role_name] task_id = domain_pinyin + '_' + role_en es_result = es.get(index=weibo_role_index_name,doc_type=weibo_role_index_type,id=task_id)['_source'] item = es_result print 'es_result:::',es_result # 政治倾向 political_side = json.loads(item['political_side'])[0][0] if political_side == 'mid': item['political_side'] = u'中立' elif political_side == 'left': item['political_side'] = u'左倾' else: item['political_side'] = u'右倾' # 心理特征 psy_feature_list = [] psy_feature = json.loads(item['psy_feature']) for i in range(TOP_PSY_FEATURE): psy_feature_list.append(psy_feature[i][0]) item['psy_feature'] = '&'.join(psy_feature_list) role_group_uids = json.loads(item['member_uids']) mget_results = es_user_portrait.mget(index=portrait_index_name,doc_type=portrait_index_type,body={'ids':role_group_uids})['docs'] # topic_list = [] # for mget_item in mget_results: # if mget_item['found']: # keywords_list = json.loads(mget_item['_source']['keywords']) # topic_list.extend(keywords_list) # topic_keywords_dict = {} # for topic_item in topic_list: # keyword = topic_item[0] # keyword_count = topic_item[1] # try: # topic_keywords_dict[keyword] += keyword_count # except: # topic_keywords_dict[keyword] = keyword_count # monitor_keywords_list = [] # for i in range(3): # keyword_max = max(topic_keywords_dict,key=topic_keywords_dict.get) # monitor_keywords_list.append(keyword_max) # del topic_keywords_dict[keyword_max] # item['monitor_keywords'] = '&'.join(monitor_keywords_list) if S_TYPE == 'test': current_time = datetime2ts(S_DATE) else: current_time = int(time.time()) index_name_list = get_flow_text_index_list(current_time) query_body_search = { 'query':{ 'filtered':{ 'filter':{ 'terms':{'uid':role_group_uids} } } }, 'size':MAX_VALUE, '_source':['keywords_string'] } es_keyword_results = es_flow_text.search(index=index_name_list,doc_type=flow_text_index_type,\ body=query_body_search)['hits']['hits'] keywords_string = '' for mget_item in es_keyword_results: #print 'mget_item:::',mget_item #if mget_item['found']: keywords_string += '&' keywords_string += mget_item['_source']['keywords_string'] k_dict = extract_keywords(keywords_string) monitor_keywords_list = [] for item_item in k_dict: monitor_keywords_list.append(item_item.word.encode('utf-8')) item['monitor_keywords'] = ','.join(monitor_keywords_list) mget_results_user = es_user_portrait.mget(index=profile_index_name,doc_type=profile_index_type,body={'ids':role_group_uids})['docs'] item['nick_name'] = [] for mget_item in mget_results_user: #print 'mget_item:::',mget_item if mget_item['found']: item['nick_name'] = mget_item['_source']['nick_name'] item['location'] = mget_item['_source']['user_location'] item['gender'] = mget_item['_source']['sex'] uid = mget_item['_source']['uid'] try: profile_results = es_user_portrait.get(index=profile_index_name,doc_type=profile_index_type,id=uid)['_source'] if profile_results['description']: item['description'] = profile_results['description'] break except: pass item['business_goal'] = u'渗透' item['daily_interests'] = u'旅游' # if S_TYPE == 'test': # user_mget_results = es.mget(index=profile_index_name,doc_type=profile_index_type,body={'ids':role_group_uids})['docs'] # if user_mget_results item['age'] = 30 item['career'] = u'自由职业' active_time_list_np = np.array(json.loads(item['active_time'])) active_time_list_np_sort = np.argsort(-active_time_list_np)[:TOP_ACTIVE_TIME] item['active_time'] = active_time_list_np_sort.tolist() day_post_num_list = np.array(json.loads(item['day_post_num'])) item['day_post_num'] = np.mean(day_post_num_list).tolist() item['role_name'] = role_name task_id_new =domain_pinyin + '_' + role_en example_model_file_name = EXAMPLE_MODEL_PATH + task_id_new + '.json' try: with open(example_model_file_name,"w") as dump_f: json.dump(item,dump_f) item_dict = dict() #item_dict['xnr_user_no'] = xnr_user_no item_dict['domain_name'] = domain_name item_dict['role_name'] = role_name es.index(index=weibo_example_model_index_name,doc_type=weibo_example_model_index_type,\ body=item_dict,id=task_id_new) mark = True except: mark = False return mark
def get_community_content(now_time, uid_list, order_by): #step1:获取uid_list #step2:生成话题词云、敏感词云、敏感帖子 query_body = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'terms': { 'uid': uid_list } }] } } } }, 'aggs': { 'sensitive_words_string': { 'terms': { 'field': 'sensitive_words_string', 'size': 800 } } }, 'size': 50, 'sort': { order_by: { 'order': 'desc' } } } query_hashtag = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'terms': { 'uid': uid_list } }] } } } }, 'aggs': { 'hashtag': { 'terms': { 'field': 'hashtag', 'size': 800 } } }, 'size': 50, 'sort': { order_by: { 'order': 'desc' } } } flow_text_index_name_list = get_flow_text_index_list(now_time) flow_text_exist = es_flow_text.search(index = flow_text_index_name_list,doc_type = flow_text_index_type,\ body = query_body) sensitive_keywords_result = flow_text_exist['aggregations'][ 'sensitive_words_string']['buckets'] content_result = flow_text_exist['hits']['hits'] hashtag_result = es_flow_text.search(index = flow_text_index_name_list,doc_type = flow_text_index_type,\ body = query_hashtag)['aggregations']['hashtag']['buckets'] # print 'content_result:',content_result # print 'sensitive_keywords_result:',sensitive_keywords_result # print 'hashtag_result:',hashtag_result sensitive_keywords_dict = get_word_count(sensitive_keywords_result) hashtag_dict = get_word_count(hashtag_result) content_list = [] for item in content_result: content_list.append(item['_source']) result_dict = dict() result_dict['topic_wordcloud'] = hashtag_dict result_dict['sensitive_wordcloud'] = sensitive_keywords_dict result_dict['content_post'] = content_list return result_dict
def get_recommend_follows(task_detail): recommend_results = dict() # daily_interests_list = task_detail['daily_interests'].split(',') monitor_keywords_list = task_detail['monitor_keywords'].split(',') create_time = time.time() if S_TYPE == 'test': create_time = datetime2ts(S_DATE) index_name_list = get_flow_text_index_list(create_time) '''#FB flow_text中没有daily_interests字段 ## 日常兴趣关注 try: query_body = { 'query':{ 'filtered':{ 'filter':{ 'terms':{'daily_interests':daily_interests_list} } } }, # 'sort':{'user_fansnum':{'order':'desc'}}, 'size':DAILY_INTEREST_TOP_USER, '_source':['uid'] } es_results = es_flow_text.search(index=index_name_list,doc_type='text',body=query_body)['hits']['hits'] daily_interest_uid_set = set() for result in es_results: daily_interest_uid_set.add(result['_source']['uid']) daily_interest_uid_list = list(daily_interest_uid_set) es_daily_interests_results = es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,\ body={'ids':daily_interest_uid_list})['docs'] nick_name_dict = {} es_daily_interests_results = es_daily_interests_results[:max(NICK_NAME_TOP,len(es_daily_interests_results))] for result in es_daily_interests_results: if result['found'] == True: result = result['_source'] nick_name_dict[result['uid']] = result['nick_name'] else: continue recommend_results['daily_interests'] = nick_name_dict except Exception,e: print e print '没有找到日常兴趣相符的用户' recommend_results['daily_interests'] = {} ''' ## 监测词关注 nest_query_list = [] #文本中可能存在英文或者繁体字,所以都匹配一下 monitor_en_keywords_list = trans(monitor_keywords_list, target_language='en') for i in range(len(monitor_keywords_list)): monitor_keyword = monitor_keywords_list[i] monitor_traditional_keyword = simplified2traditional(monitor_keyword) if len(monitor_en_keywords_list) == len(monitor_keywords_list): #确保翻译没出错 monitor_en_keyword = monitor_en_keywords_list[i] nest_query_list.append({'wildcard':{'keywords_string':'*'+monitor_en_keyword+'*'}}) nest_query_list.append({'wildcard':{'keywords_string':'*'+monitor_keyword+'*'}}) nest_query_list.append({'wildcard':{'keywords_string':'*'+monitor_traditional_keyword+'*'}}) try: query_body_monitor = { 'query':{ 'bool':{ # 'must':nest_query_list 'should':nest_query_list } }, # 'sort':{'user_fansnum':{'order':'desc'}}, 'size':MONITOR_TOP_USER, '_source':['uid'] } es_results = es_flow_text.search(index=index_name_list,doc_type='text',body=query_body_monitor)['hits']['hits'] monitor_keywords_uid_set = set() for result in es_results: monitor_keywords_uid_set.add(result['_source']['uid']) monitor_keywords_uid_list = list(monitor_keywords_uid_set) es_monitor_keywords_results = es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,\ body={'ids':monitor_keywords_uid_list})['docs'] nick_name_dict = {} es_monitor_keywords_results = es_monitor_keywords_results[:max(NICK_NAME_TOP,len(es_monitor_keywords_results))] for result in es_monitor_keywords_results: if result['found'] == True: result = result['_source'] nick_name_dict[result['uid']] = result['username'] else: continue recommend_results['monitor_keywords'] = nick_name_dict except Exception,e: print e print '没有找到监测词相符的用户' recommend_results['monitor_keywords'] = {}
def get_recommend_follows(task_detail): recommend_results = dict() daily_interests_list = task_detail['daily_interests'].encode('utf-8').split(',') monitor_keywords_list = task_detail['monitor_keywords'].encode('utf-8').split(',') #print 'daily_interests_list::',daily_interests_list create_time = time.time() if S_TYPE == 'test': create_time = datetime2ts(S_DATE) index_name_list = get_flow_text_index_list(create_time) ## 日常兴趣关注 try: query_body = { 'query':{ 'filtered':{ 'filter':{ 'terms':{'daily_interests':daily_interests_list} } } }, 'sort':{'user_fansnum':{'order':'desc'}}, 'size':DAILY_INTEREST_TOP_USER, '_source':['uid'] } es_results = es_flow_text.search(index=index_name_list,doc_type='text',body=query_body)['hits']['hits'] daily_interest_uid_set = set() for result in es_results: daily_interest_uid_set.add(result['_source']['uid']) daily_interest_uid_list = list(daily_interest_uid_set) es_daily_interests_results = es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,\ body={'ids':daily_interest_uid_list})['docs'] nick_name_dict = {} es_daily_interests_results = es_daily_interests_results[:max(NICK_NAME_TOP,len(es_daily_interests_results))] for result in es_daily_interests_results: if result['found'] == True: result = result['_source'] nick_name_dict[result['uid']] = result['nick_name'] else: continue recommend_results['daily_interests'] = nick_name_dict except: print '没有找到日常兴趣相符的用户' recommend_results['daily_interests'] = {} ## 监测词关注 nest_query_list = [] #print 'monitor_keywords_list:::',monitor_keywords_list for monitor_keyword in monitor_keywords_list: nest_query_list.append({'wildcard':{'keywords_string':'*'+monitor_keyword+'*'}}) #print 'nest_query_list::',nest_query_list try: query_body_monitor = { 'query':{ 'bool':{ 'must':nest_query_list } }, 'sort':{'user_fansnum':{'order':'desc'}}, 'size':MONITOR_TOP_USER, '_source':['uid'] } #print '123' es_results = es_flow_text.search(index=index_name_list,doc_type='text',body=query_body_monitor)['hits']['hits'] #print 'es_results::',es_results monitor_keywords_uid_set = set() for result in es_results: monitor_keywords_uid_set.add(result['_source']['uid']) monitor_keywords_uid_list = list(monitor_keywords_uid_set) es_monitor_keywords_results = es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,\ body={'ids':monitor_keywords_uid_list})['docs'] nick_name_dict = {} es_monitor_keywords_results = es_monitor_keywords_results[:max(NICK_NAME_TOP,len(es_monitor_keywords_results))] for result in es_monitor_keywords_results: if result['found'] == True: result = result['_source'] nick_name_dict[result['uid']] = result['nick_name'] else: continue recommend_results['monitor_keywords'] = nick_name_dict except: print '没有找到监测词相符的用户' recommend_results['monitor_keywords'] = {} # print 'recommend_results::',recommend_results return recommend_results
def get_generate_example_model(domain_name,role_name): domain_pinyin = pinyin.get(domain_name,format='strip',delimiter='_') role_en = fb_domain_ch2en_dict[role_name] task_id = domain_pinyin + '_' + role_en es_result = es.get(index=fb_role_index_name,doc_type=fb_role_index_type,id=task_id)['_source'] item = es_result print 'es_result:::',es_result # 政治倾向 political_side = json.loads(item['political_side'])[0][0] if political_side == 'mid': item['political_side'] = u'中立' elif political_side == 'left': item['political_side'] = u'左倾' else: item['political_side'] = u'右倾' # 心理特征 psy_feature_list = [] psy_feature = json.loads(item['psy_feature']) for i in range(TOP_PSY_FEATURE): psy_feature_list.append(psy_feature[i][0]) item['psy_feature'] = '&'.join(psy_feature_list) role_group_uids = json.loads(item['member_uids']) if S_TYPE == 'test': current_time = datetime2ts(S_DATE) else: current_time = int(time.time()) index_name_list = get_flow_text_index_list(current_time) query_body_search = { 'query':{ 'filtered':{ 'filter':{ 'terms':{'uid':role_group_uids} } } }, 'size':MAX_VALUE, '_source':['keywords_string'] } es_keyword_results = es_flow_text.search(index=index_name_list,doc_type=flow_text_index_type,\ body=query_body_search)['hits']['hits'] keywords_string = '' for mget_item in es_keyword_results: keywords_string += '&' keywords_string += mget_item['_source']['keywords_string'] k_dict = extract_keywords(keywords_string) monitor_keywords_list = [] for item_item in k_dict: monitor_keywords_list.append(item_item.word.encode('utf-8')) item['monitor_keywords'] = ','.join(monitor_keywords_list) mget_results_user = es_user_portrait.mget(index=profile_index_name,doc_type=profile_index_type,body={'ids':role_group_uids})['docs'] item['nick_name'] = [] for mget_item in mget_results_user: if mget_item['found']: content = mget_item['_source'] item['nick_name'] = '' if content.has_key('name'): item['nick_name'] = content['name'] item['location'] = '' if content.has_key('location'): item['location'] = get_user_location(json.loads(content['location'])) item['gender'] = 0 if content.has_key('gender'): gender_str = content['gender'] if gender_str == 'male': gender = 1 elif gender_str == 'female': gender = 2 item['description'] = '' if content.has_key('description'): item['description'] = content['description'] item['business_goal'] = u'渗透' item['daily_interests'] = u'旅游' item['age'] = 30 item['career'] = u'自由职业' active_time_list_np = np.array(json.loads(item['active_time'])) active_time_list_np_sort = np.argsort(-active_time_list_np)[:TOP_ACTIVE_TIME] item['active_time'] = active_time_list_np_sort.tolist() day_post_num_list = np.array(json.loads(item['day_post_num'])) item['day_post_num'] = np.mean(day_post_num_list).tolist() item['role_name'] = role_name task_id_new = 'fb_' + domain_pinyin + '_' + role_en example_model_file_name = EXAMPLE_MODEL_PATH + task_id_new + '.json' try: with open(example_model_file_name,"w") as dump_f: json.dump(item,dump_f) item_dict = dict() item_dict['domain_name'] = domain_name item_dict['role_name'] = role_name es.index(index=fb_example_model_index_name,doc_type=fb_example_model_index_type,\ body=item_dict,id=task_id_new) mark = True except: mark = False return mark
def show_event_warming(xnr_user_no): now_time = int(time.time()) #print 'first_time:',time.time() hashtag_list = get_hashtag() #print 'hashtag_list_time:',time.time() #print 'hashtag_list:::::::',hashtag_list if S_TYPE == 'test': test_day_date = S_DATE_EVENT_WARMING test_day_time = datetime2ts(test_day_date) flow_text_index_list = get_flow_text_index_list(test_day_time) #print flow_text_index_list hashtag_list = [('网络义勇军发布', 13), ('美国', 7), ('德国', 5), ('中国', 4), ('清真食品', 3), ('反邪动态', 2), ('台海观察', 2), ('雷哥微评', 2), ('中国军队', 1)] #hashtag_list=[('网络义勇军发布',13),('美国',7),('芒果TV',6),('德国',5),('中国',4),('清真食品',3),('反邪动态',2),('台海观察',2),('每日一药',2),('雷哥微评',2),('PPAP洗脑神曲',1),('中国军队',1)] #weibo_xnr_flow_text_listname=['flow_text_2016-11-26','flow_text_2016-11-25','flow_text_2016-11-24'] else: flow_text_index_list = get_flow_text_index_list(now_time) #weibo_xnr_flow_text_listname=get_xnr_flow_text_index_list(now_time) #print flow_text_index_list,hashtag_list #虚拟人的粉丝列表和关注列表 try: es_xnr_result = es_xnr.get( index=weibo_xnr_fans_followers_index_name, doc_type=weibo_xnr_fans_followers_index_type, id=xnr_user_no)['_source'] followers_list = es_xnr_result['followers_list'] fans_list = es_xnr_result['fans_list'] except: followers_list = [] fans_list = [] #print 'weibo_xnr_fans_followers_time:',time.time() event_warming_list = [] for event_item in hashtag_list: #print event_item,event_item[0] event_sensitive_count = 0 event_warming_content = dict() #事件名称、主要参与用户、典型微博、事件影响力、事件平均时间 event_warming_content['event_name'] = event_item[0] event_influence_sum = 0 event_time_sum = 0 query_body = { 'query': { 'bool': { 'should': { 'wildcard': { 'text': '*' + event_item[0] + '*' } } } } } try: event_results = es_flow_text.search( index=flow_text_index_list, doc_type=flow_text_index_type, body=query_body)['hits']['hits'] weibo_result = [] fans_num_dict = dict() followers_num_dict = dict() alluser_num_dict = dict() #print event_results for item in event_results: if item['_source']['sensitive'] > 0: event_sensitive_count = event_sensitive_count + 1 #统计用户信息 if alluser_num_dict.has_key(str(item['_source']['uid'])): alluser_num_dict[str( item['_source']['uid'])] = alluser_num_dict[str( item['_source']['uid'])] + 1 else: alluser_num_dict[str(item['_source']['uid'])] = 1 for fans_uid in fans_list: if fans_uid == item['_source']['uid']: if fans_num_dict.has_key(str(fans_uid)): fans_num_dict[str(fans_uid)] = fans_num_dict[ str(fans_uid)] + 1 else: fans_num_dict[str(fans_uid)] = 1 for followers_uid in followers_list: if followers_uid == item['_source']['uid']: if followers_num_dict.has_key(str(followers_uid)): fans_num_dict[str( followers_uid )] = fans_num_dict[str(followers_uid)] + 1 else: fans_num_dict[str(followers_uid)] = 1 #计算影响力 origin_influence_value = (item['_source']['comment'] + item['_source']['retweeted']) * ( 1 + item['_source']['sensitive']) fans_value = judge_user_type(item['_source']['uid'], fans_list) followers_value = judge_user_type(item['_source']['uid'], followers_list) item['_source'][ 'weibo_influence_value'] = origin_influence_value * ( fans_value + followers_value) weibo_result.append(item['_source']) #统计影响力、时间 event_influence_sum = event_influence_sum + item[ '_source']['weibo_influence_value'] event_time_sum = item['_source']['timestamp'] #典型微博信息 weibo_result.sort(key=lambda k: (k.get('weibo_influence_value', 0)), reverse=True) event_warming_content['main_weibo_info'] = weibo_result #事件影响力和事件时间 number = len(event_results) event_warming_content[ 'event_influence'] = event_influence_sum / number event_warming_content['event_time'] = event_time_sum / number else: pass except: event_warming_content['main_weibo_info'] = [] event_warming_content['event_influence'] = [] event_warming_content['event_time'] = [] #print event_item[0],'event_search_time:',time.time() try: if event_sensitive_count > 0: #对用户进行排序 temp_userid_dict = union_dict(fans_num_dict, followers_num_dict) main_userid_dict = union_dict(temp_userid_dict, alluser_num_dict) main_userid_dict = sorted(main_userid_dict.items(), key=lambda d: d[1], reverse=True) main_userid_list = [] for i in xrange(0, len(main_userid_dict)): main_userid_list.append(main_userid_dict[i][0]) #print 'main_userid_list:',main_userid_list #主要参与用户信息 main_user_info = [] user_es_result = es_user_profile.mget( index=profile_index_name, doc_type=profile_index_type, body={'ids': main_userid_list})['docs'] for item in user_es_result: #print 'item:',item #print 'found:',item['found'] #print 'id:',item['_id'] user_dict = dict() if item['found']: user_dict['photo_url'] = item['_source']['photo_url'] user_dict['uid'] = item['_id'] user_dict['nick_name'] = item['_source']['nick_name'] user_dict['favoritesnum'] = item['_source'][ 'favoritesnum'] user_dict['fansnum'] = item['_source']['fansnum'] else: user_dict['photo_url'] = '' user_dict['uid'] = item['_id'] user_dict['nick_name'] = '' user_dict['favoritesnum'] = 0 user_dict['fansnum'] = 0 main_user_info.append(user_dict) event_warming_content['main_user_info'] = main_user_info #print 'main_user_info:',main_user_info #print user_es_result ''' user_query_body={ 'query':{ 'filtered':{ 'filter':{ 'terms':{'uid':main_userid_list} } } } } user_es_result=es_user_profile.search(index=profile_index_name,doc_type=profile_index_type,body=user_query_body)['hits']['hits'] #print user_es_result main_user_info=[] for item in user_es_result: main_user_info.append(item['_source']) event_warming_content['main_user_info']=main_user_info ''' else: event_warming_content['main_user_info'] = [] except: event_warming_content['main_user_info'] = [] #print 'user_search_time:',time.time() if event_sensitive_count > 0: #print event_warming_content['event_name'] event_warming_list.append(event_warming_content) else: pass #main_userid_list=['5536381570','2192435767','1070598590'] #user_es_result=es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,body={'ids':main_userid_list}) #print 'user_es_result',user_es_result #print 'end_time:',time.time() return event_warming_list
def show_personnal_warming(xnr_user_no, day_time): #查询关注列表 try: es_xnr_result = es_xnr.get( index=weibo_xnr_fans_followers_index_name, doc_type=weibo_xnr_fans_followers_index_type, id=xnr_user_no)['_source'] followers_list = es_xnr_result['followers_list'] except: followers_list = [] #计算敏感度排名靠前的用户 query_body = { 'query': { 'filtered': { 'filter': { 'terms': { 'uid': followers_list } } } }, 'aggs': { 'followers_sensitive_num': { 'terms': { 'field': 'uid' }, 'aggs': { 'sensitive_num': { 'sum': { 'field': 'sensitive' } } } } }, 'size': MAX_VALUE } #测试状态下时间设置 if S_TYPE == 'test': test_day_date = S_DATE_BCI test_day_time = datetime2ts(test_day_date) flow_text_index_list = get_flow_text_index_list(test_day_time) else: flow_text_index_list = get_flow_text_index_list(day_time) #print flow_text_index_list try: first_sum_result=es_flow_text.search(index=flow_text_index_list,doc_type=flow_text_index_type,\ body=query_body)['aggregations']['followers_sensitive_num']['buckets'] except: first_sum_result = [] #print first_sum_result top_userlist = [] if USER_NUM < len(first_sum_result): temp_num = USER_NUM else: temp_num = len(first_sum_result) #print temp_num for i in xrange(0, temp_num): user_sensitive = first_sum_result[i]['sensitive_num']['value'] if user_sensitive > 0: user_dict = dict() user_dict['uid'] = first_sum_result[i]['key'] user_dict['sensitive'] = user_sensitive top_userlist.append(user_dict) else: pass #查询敏感用户的最敏感微博内容 results = [] for user in top_userlist: #print user user_detail = dict() user_detail['uid'] = user['uid'] user_detail['user_sensitive'] = user['sensitive'] try: user_result = es_user_profile.get(index=profile_index_name, doc_type=profile_index_type, id=user['uid'])['_source'] user_detail['user_name'] = user_result['nick_name'] except: user_detail['user_name'] = '' query_body = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'term': { 'uid': user['uid'] } }, { 'range': { 'sensitive': { 'gte': 1, 'lte': 100 } } }] } } } }, 'size': USER_CONTENT_NUM, 'sort': { 'sensitive': { 'order': 'desc' } } } #if S_TYPE == 'test': try: second_result = es_flow_text.search( index=flow_text_index_list, doc_type=flow_text_index_type, body=query_body)['hits']['hits'] except: second_result = [] #else: # second_result=es_xnr.search(index=weibo_xnr_flow_text_listname,doc_type=xnr_flow_text_index_type,body=query_body)['hits']['hits'] s_result = [] tem_word_one = '静坐' tem_word_two = '集合' for item in second_result: sensitive_words = item['_source']['sensitive_words_string'] if ((sensitive_words == tem_word_one) or (sensitive_words == tem_word_two)): pass else: s_result.append(item['_source']) s_result.sort(key=lambda k: (k.get('sensitive', 0)), reverse=True) user_detail['content'] = s_result results.append(user_detail) results.sort(key=lambda k: (k.get('user_sensitive', 0)), reverse=True) return results
def show_speech_warming(xnr_user_no, show_type, day_time): #关注用户 try: es_xnr_result = es_xnr.get( index=weibo_xnr_fans_followers_index_name, doc_type=weibo_xnr_fans_followers_index_type, id=xnr_user_no)['_source'] followers_list = es_xnr_result['followers_list'] except: followers_list = [] show_condition_list = [] if show_type == 0: #全部用户 show_condition_list.append( {'must': { 'range': { 'sensitive': { 'gte': 1, 'lte': 100 } } }}) elif show_type == 1: #关注用户 show_condition_list.append({ 'must': [{ 'terms': { 'uid': followers_list } }, { 'range': { 'sensitive': { 'gte': 1, 'lte': 100 } } }] }) elif show_type == 2: #未关注用户 show_condition_list.append({ 'must_not': { 'terms': { 'uid': followers_list } }, 'must': { 'range': { 'sensitive': { 'gte': 1, 'lte': 100 } } } }) query_body = { 'query': { 'filtered': { 'filter': { 'bool': show_condition_list[0] } } }, 'size': SPEECH_WARMING_NUM, 'sort': { 'sensitive': { 'order': 'desc' } } } #测试状态下时间设置 if S_TYPE == 'test': test_day_date = S_DATE_BCI test_day_time = datetime2ts(test_day_date) flow_text_index_list = get_flow_text_index_list(test_day_time) else: flow_text_index_list = get_flow_text_index_list(day_time) #try: results = es_flow_text.search(index=flow_text_index_list, doc_type=flow_text_index_type, body=query_body)['hits']['hits'] result = [] un_id_list = [ '4045093692450438', '4045096116622444', '4045095374193153', '4045095567336676', '4045092304116237', '4045093297982719', '4045178576337277', '4044647661388452' ] for item in results: if item['_id'] in un_id_list: pass else: result.append(item['_source']) #except: # result=[] return result