def get_generate_example_model(domain_name,role_name):
domain_pinyin = pinyin.get(domain_name,format='strip',delimiter='_')
role_en = domain_ch2en_dict[role_name]
task_id = domain_pinyin + '_' + role_en
es_result = es.get(index=weibo_role_index_name,doc_type=weibo_role_index_type,id=task_id)['_source']
item = es_result
print 'es_result:::',es_result
# 政治倾向
political_side = json.loads(item['political_side'])[0][0]
if political_side == 'mid':
item['political_side'] = u'中立'
elif political_side == 'left':
item['political_side'] = u'左倾'
else:
item['political_side'] = u'右倾'
# 心理特征
psy_feature_list = []
psy_feature = json.loads(item['psy_feature'])
for i in range(TOP_PSY_FEATURE):
psy_feature_list.append(psy_feature[i][0])
item['psy_feature'] = '&'.join(psy_feature_list)
role_group_uids = json.loads(item['member_uids'])
mget_results = es_user_portrait.mget(index=portrait_index_name,doc_type=portrait_index_type,body={'ids':role_group_uids})['docs']
# topic_list = []
# for mget_item in mget_results:
# if mget_item['found']:
# keywords_list = json.loads(mget_item['_source']['keywords'])
# topic_list.extend(keywords_list)
# topic_keywords_dict = {}
# for topic_item in topic_list:
# keyword = topic_item[0]
# keyword_count = topic_item[1]
# try:
# topic_keywords_dict[keyword] += keyword_count
# except:
# topic_keywords_dict[keyword] = keyword_count
# monitor_keywords_list = []
# for i in range(3):
# keyword_max = max(topic_keywords_dict,key=topic_keywords_dict.get)
# monitor_keywords_list.append(keyword_max)
# del topic_keywords_dict[keyword_max]
# item['monitor_keywords'] = '&'.join(monitor_keywords_list)
if S_TYPE == 'test':
current_time = datetime2ts(S_DATE)
else:
current_time = int(time.time())
index_name_list = get_flow_text_index_list(current_time)
query_body_search = {
'query':{
'filtered':{
'filter':{
'terms':{'uid':role_group_uids}
}
}
},
'size':MAX_VALUE,
'_source':['keywords_string']
}
es_keyword_results = es_flow_text.search(index=index_name_list,doc_type=flow_text_index_type,\
body=query_body_search)['hits']['hits']
keywords_string = ''
for mget_item in es_keyword_results:
#print 'mget_item:::',mget_item
#if mget_item['found']:
keywords_string += '&'
keywords_string += mget_item['_source']['keywords_string']
k_dict = extract_keywords(keywords_string)
monitor_keywords_list = []
for item_item in k_dict:
monitor_keywords_list.append(item_item.word.encode('utf-8'))
item['monitor_keywords'] = ','.join(monitor_keywords_list)
mget_results_user = es_user_portrait.mget(index=profile_index_name,doc_type=profile_index_type,body={'ids':role_group_uids})['docs']
item['nick_name'] = []
for mget_item in mget_results_user:
#print 'mget_item:::',mget_item
if mget_item['found']:
item['nick_name'] = mget_item['_source']['nick_name']
item['location'] = mget_item['_source']['user_location']
item['gender'] = mget_item['_source']['sex']
uid = mget_item['_source']['uid']
try:
profile_results = es_user_portrait.get(index=profile_index_name,doc_type=profile_index_type,id=uid)['_source']
if profile_results['description']:
item['description'] = profile_results['description']
break
except:
pass
item['business_goal'] = u'渗透'
item['daily_interests'] = u'旅游'
# if S_TYPE == 'test':
# user_mget_results = es.mget(index=profile_index_name,doc_type=profile_index_type,body={'ids':role_group_uids})['docs']
# if user_mget_results
item['age'] = 30
item['career'] = u'自由职业'
active_time_list_np = np.array(json.loads(item['active_time']))
active_time_list_np_sort = np.argsort(-active_time_list_np)[:TOP_ACTIVE_TIME]
item['active_time'] = active_time_list_np_sort.tolist()
day_post_num_list = np.array(json.loads(item['day_post_num']))
item['day_post_num'] = np.mean(day_post_num_list).tolist()
item['role_name'] = role_name
task_id_new =domain_pinyin + '_' + role_en
example_model_file_name = EXAMPLE_MODEL_PATH + task_id_new + '.json'
try:
with open(example_model_file_name,"w") as dump_f:
json.dump(item,dump_f)
item_dict = dict()
#item_dict['xnr_user_no'] = xnr_user_no
item_dict['domain_name'] = domain_name
item_dict['role_name'] = role_name
es.index(index=weibo_example_model_index_name,doc_type=weibo_example_model_index_type,\
body=item_dict,id=task_id_new)
mark = True
except:
mark = False
return mark
def get_community_content(now_time, uid_list, order_by):
#step1:获取uid_list
#step2:生成话题词云、敏感词云、敏感帖子
query_body = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'terms': {
'uid': uid_list
}
}]
}
}
}
},
'aggs': {
'sensitive_words_string': {
'terms': {
'field': 'sensitive_words_string',
'size': 800
}
}
},
'size': 50,
'sort': {
order_by: {
'order': 'desc'
}
}
}
query_hashtag = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'terms': {
'uid': uid_list
}
}]
}
}
}
},
'aggs': {
'hashtag': {
'terms': {
'field': 'hashtag',
'size': 800
}
}
},
'size': 50,
'sort': {
order_by: {
'order': 'desc'
}
}
}
flow_text_index_name_list = get_flow_text_index_list(now_time)
flow_text_exist = es_flow_text.search(index = flow_text_index_name_list,doc_type = flow_text_index_type,\
body = query_body)
sensitive_keywords_result = flow_text_exist['aggregations'][
'sensitive_words_string']['buckets']
content_result = flow_text_exist['hits']['hits']
hashtag_result = es_flow_text.search(index = flow_text_index_name_list,doc_type = flow_text_index_type,\
body = query_hashtag)['aggregations']['hashtag']['buckets']
# print 'content_result:',content_result
# print 'sensitive_keywords_result:',sensitive_keywords_result
# print 'hashtag_result:',hashtag_result
sensitive_keywords_dict = get_word_count(sensitive_keywords_result)
hashtag_dict = get_word_count(hashtag_result)
content_list = []
for item in content_result:
content_list.append(item['_source'])
result_dict = dict()
result_dict['topic_wordcloud'] = hashtag_dict
result_dict['sensitive_wordcloud'] = sensitive_keywords_dict
result_dict['content_post'] = content_list
return result_dict
def get_recommend_follows(task_detail):
recommend_results = dict()
# daily_interests_list = task_detail['daily_interests'].split('，')
monitor_keywords_list = task_detail['monitor_keywords'].split('，')
create_time = time.time()
if S_TYPE == 'test':
create_time = datetime2ts(S_DATE)
index_name_list = get_flow_text_index_list(create_time)
'''#FB flow_text中没有daily_interests字段
## 日常兴趣关注
try:
query_body = {
'query':{
'filtered':{
'filter':{
'terms':{'daily_interests':daily_interests_list}
}
}
},
# 'sort':{'user_fansnum':{'order':'desc'}},
'size':DAILY_INTEREST_TOP_USER,
'_source':['uid']
}
es_results = es_flow_text.search(index=index_name_list,doc_type='text',body=query_body)['hits']['hits']
daily_interest_uid_set = set()
for result in es_results:
daily_interest_uid_set.add(result['_source']['uid'])
daily_interest_uid_list = list(daily_interest_uid_set)
es_daily_interests_results = es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,\
body={'ids':daily_interest_uid_list})['docs']
nick_name_dict = {}
es_daily_interests_results = es_daily_interests_results[:max(NICK_NAME_TOP,len(es_daily_interests_results))]
for result in es_daily_interests_results:
if result['found'] == True:
result = result['_source']
nick_name_dict[result['uid']] = result['nick_name']
else:
continue
recommend_results['daily_interests'] = nick_name_dict
except Exception,e:
print e
print '没有找到日常兴趣相符的用户'
recommend_results['daily_interests'] = {}
'''
## 监测词关注
nest_query_list = []
#文本中可能存在英文或者繁体字，所以都匹配一下
monitor_en_keywords_list = trans(monitor_keywords_list, target_language='en')
for i in range(len(monitor_keywords_list)):
monitor_keyword = monitor_keywords_list[i]
monitor_traditional_keyword = simplified2traditional(monitor_keyword)
if len(monitor_en_keywords_list) == len(monitor_keywords_list): #确保翻译没出错
monitor_en_keyword = monitor_en_keywords_list[i]
nest_query_list.append({'wildcard':{'keywords_string':'*'+monitor_en_keyword+'*'}})
nest_query_list.append({'wildcard':{'keywords_string':'*'+monitor_keyword+'*'}})
nest_query_list.append({'wildcard':{'keywords_string':'*'+monitor_traditional_keyword+'*'}})
try:
query_body_monitor = {
'query':{
'bool':{
# 'must':nest_query_list
'should':nest_query_list
}
},
# 'sort':{'user_fansnum':{'order':'desc'}},
'size':MONITOR_TOP_USER,
'_source':['uid']
}
es_results = es_flow_text.search(index=index_name_list,doc_type='text',body=query_body_monitor)['hits']['hits']
monitor_keywords_uid_set = set()
for result in es_results:
monitor_keywords_uid_set.add(result['_source']['uid'])
monitor_keywords_uid_list = list(monitor_keywords_uid_set)
es_monitor_keywords_results = es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,\
body={'ids':monitor_keywords_uid_list})['docs']
nick_name_dict = {}
es_monitor_keywords_results = es_monitor_keywords_results[:max(NICK_NAME_TOP,len(es_monitor_keywords_results))]
for result in es_monitor_keywords_results:
if result['found'] == True:
result = result['_source']
nick_name_dict[result['uid']] = result['username']
else:
continue
recommend_results['monitor_keywords'] = nick_name_dict
except Exception,e:
print e
print '没有找到监测词相符的用户'
recommend_results['monitor_keywords'] = {}
def get_recommend_follows(task_detail):
recommend_results = dict()
daily_interests_list = task_detail['daily_interests'].encode('utf-8').split('，')
monitor_keywords_list = task_detail['monitor_keywords'].encode('utf-8').split('，')
#print 'daily_interests_list::',daily_interests_list
create_time = time.time()
if S_TYPE == 'test':
create_time = datetime2ts(S_DATE)
index_name_list = get_flow_text_index_list(create_time)
## 日常兴趣关注
try:
query_body = {
'query':{
'filtered':{
'filter':{
'terms':{'daily_interests':daily_interests_list}
}
}
},
'sort':{'user_fansnum':{'order':'desc'}},
'size':DAILY_INTEREST_TOP_USER,
'_source':['uid']
}
es_results = es_flow_text.search(index=index_name_list,doc_type='text',body=query_body)['hits']['hits']
daily_interest_uid_set = set()
for result in es_results:
daily_interest_uid_set.add(result['_source']['uid'])
daily_interest_uid_list = list(daily_interest_uid_set)
es_daily_interests_results = es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,\
body={'ids':daily_interest_uid_list})['docs']
nick_name_dict = {}
es_daily_interests_results = es_daily_interests_results[:max(NICK_NAME_TOP,len(es_daily_interests_results))]
for result in es_daily_interests_results:
if result['found'] == True:
result = result['_source']
nick_name_dict[result['uid']] = result['nick_name']
else:
continue
recommend_results['daily_interests'] = nick_name_dict
except:
print '没有找到日常兴趣相符的用户'
recommend_results['daily_interests'] = {}
## 监测词关注
nest_query_list = []
#print 'monitor_keywords_list:::',monitor_keywords_list
for monitor_keyword in monitor_keywords_list:
nest_query_list.append({'wildcard':{'keywords_string':'*'+monitor_keyword+'*'}})
#print 'nest_query_list::',nest_query_list
try:
query_body_monitor = {
'query':{
'bool':{
'must':nest_query_list
}
},
'sort':{'user_fansnum':{'order':'desc'}},
'size':MONITOR_TOP_USER,
'_source':['uid']
}
#print '123'
es_results = es_flow_text.search(index=index_name_list,doc_type='text',body=query_body_monitor)['hits']['hits']
#print 'es_results::',es_results
monitor_keywords_uid_set = set()
for result in es_results:
monitor_keywords_uid_set.add(result['_source']['uid'])
monitor_keywords_uid_list = list(monitor_keywords_uid_set)
es_monitor_keywords_results = es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,\
body={'ids':monitor_keywords_uid_list})['docs']
nick_name_dict = {}
es_monitor_keywords_results = es_monitor_keywords_results[:max(NICK_NAME_TOP,len(es_monitor_keywords_results))]
for result in es_monitor_keywords_results:
if result['found'] == True:
result = result['_source']
nick_name_dict[result['uid']] = result['nick_name']
else:
continue
recommend_results['monitor_keywords'] = nick_name_dict
except:
print '没有找到监测词相符的用户'
recommend_results['monitor_keywords'] = {}
# print 'recommend_results::',recommend_results
return recommend_results
def get_generate_example_model(domain_name,role_name):
domain_pinyin = pinyin.get(domain_name,format='strip',delimiter='_')
role_en = fb_domain_ch2en_dict[role_name]
task_id = domain_pinyin + '_' + role_en
es_result = es.get(index=fb_role_index_name,doc_type=fb_role_index_type,id=task_id)['_source']
item = es_result
print 'es_result:::',es_result
# 政治倾向
political_side = json.loads(item['political_side'])[0][0]
if political_side == 'mid':
item['political_side'] = u'中立'
elif political_side == 'left':
item['political_side'] = u'左倾'
else:
item['political_side'] = u'右倾'
# 心理特征
psy_feature_list = []
psy_feature = json.loads(item['psy_feature'])
for i in range(TOP_PSY_FEATURE):
psy_feature_list.append(psy_feature[i][0])
item['psy_feature'] = '&'.join(psy_feature_list)
role_group_uids = json.loads(item['member_uids'])
if S_TYPE == 'test':
current_time = datetime2ts(S_DATE)
else:
current_time = int(time.time())
index_name_list = get_flow_text_index_list(current_time)
query_body_search = {
'query':{
'filtered':{
'filter':{
'terms':{'uid':role_group_uids}
}
}
},
'size':MAX_VALUE,
'_source':['keywords_string']
}
es_keyword_results = es_flow_text.search(index=index_name_list,doc_type=flow_text_index_type,\
body=query_body_search)['hits']['hits']
keywords_string = ''
for mget_item in es_keyword_results:
keywords_string += '&'
keywords_string += mget_item['_source']['keywords_string']
k_dict = extract_keywords(keywords_string)
monitor_keywords_list = []
for item_item in k_dict:
monitor_keywords_list.append(item_item.word.encode('utf-8'))
item['monitor_keywords'] = ','.join(monitor_keywords_list)
mget_results_user = es_user_portrait.mget(index=profile_index_name,doc_type=profile_index_type,body={'ids':role_group_uids})['docs']
item['nick_name'] = []
for mget_item in mget_results_user:
if mget_item['found']:
content = mget_item['_source']
item['nick_name'] = ''
if content.has_key('name'):
item['nick_name'] = content['name']
item['location'] = ''
if content.has_key('location'):
item['location'] = get_user_location(json.loads(content['location']))
item['gender'] = 0
if content.has_key('gender'):
gender_str = content['gender']
if gender_str == 'male':
gender = 1
elif gender_str == 'female':
gender = 2
item['description'] = ''
if content.has_key('description'):
item['description'] = content['description']
item['business_goal'] = u'渗透'
item['daily_interests'] = u'旅游'
item['age'] = 30
item['career'] = u'自由职业'
active_time_list_np = np.array(json.loads(item['active_time']))
active_time_list_np_sort = np.argsort(-active_time_list_np)[:TOP_ACTIVE_TIME]
item['active_time'] = active_time_list_np_sort.tolist()
day_post_num_list = np.array(json.loads(item['day_post_num']))
item['day_post_num'] = np.mean(day_post_num_list).tolist()
item['role_name'] = role_name
task_id_new = 'fb_' + domain_pinyin + '_' + role_en
example_model_file_name = EXAMPLE_MODEL_PATH + task_id_new + '.json'
try:
with open(example_model_file_name,"w") as dump_f:
json.dump(item,dump_f)
item_dict = dict()
item_dict['domain_name'] = domain_name
item_dict['role_name'] = role_name
es.index(index=fb_example_model_index_name,doc_type=fb_example_model_index_type,\
body=item_dict,id=task_id_new)
mark = True
except:
mark = False
return mark
def show_event_warming(xnr_user_no):
now_time = int(time.time())
#print 'first_time:',time.time()
hashtag_list = get_hashtag()
#print 'hashtag_list_time:',time.time()
#print 'hashtag_list:::::::',hashtag_list
if S_TYPE == 'test':
test_day_date = S_DATE_EVENT_WARMING
test_day_time = datetime2ts(test_day_date)
flow_text_index_list = get_flow_text_index_list(test_day_time)
#print flow_text_index_list
hashtag_list = [('网络义勇军发布', 13), ('美国', 7), ('德国', 5), ('中国', 4),
('清真食品', 3), ('反邪动态', 2), ('台海观察', 2), ('雷哥微评', 2),
('中国军队', 1)]
#hashtag_list=[('网络义勇军发布',13),('美国',7),('芒果TV',6),('德国',5),('中国',4),('清真食品',3),('反邪动态',2),('台海观察',2),('每日一药',2),('雷哥微评',2),('PPAP洗脑神曲',1),('中国军队',1)]
#weibo_xnr_flow_text_listname=['flow_text_2016-11-26','flow_text_2016-11-25','flow_text_2016-11-24']
else:
flow_text_index_list = get_flow_text_index_list(now_time)
#weibo_xnr_flow_text_listname=get_xnr_flow_text_index_list(now_time)
#print flow_text_index_list,hashtag_list
#虚拟人的粉丝列表和关注列表
try:
es_xnr_result = es_xnr.get(
index=weibo_xnr_fans_followers_index_name,
doc_type=weibo_xnr_fans_followers_index_type,
id=xnr_user_no)['_source']
followers_list = es_xnr_result['followers_list']
fans_list = es_xnr_result['fans_list']
except:
followers_list = []
fans_list = []
#print 'weibo_xnr_fans_followers_time:',time.time()
event_warming_list = []
for event_item in hashtag_list:
#print event_item,event_item[0]
event_sensitive_count = 0
event_warming_content = dict() #事件名称、主要参与用户、典型微博、事件影响力、事件平均时间
event_warming_content['event_name'] = event_item[0]
event_influence_sum = 0
event_time_sum = 0
query_body = {
'query': {
'bool': {
'should': {
'wildcard': {
'text': '*' + event_item[0] + '*'
}
}
}
}
}
try:
event_results = es_flow_text.search(
index=flow_text_index_list,
doc_type=flow_text_index_type,
body=query_body)['hits']['hits']
weibo_result = []
fans_num_dict = dict()
followers_num_dict = dict()
alluser_num_dict = dict()
#print event_results
for item in event_results:
if item['_source']['sensitive'] > 0:
event_sensitive_count = event_sensitive_count + 1
#统计用户信息
if alluser_num_dict.has_key(str(item['_source']['uid'])):
alluser_num_dict[str(
item['_source']['uid'])] = alluser_num_dict[str(
item['_source']['uid'])] + 1
else:
alluser_num_dict[str(item['_source']['uid'])] = 1
for fans_uid in fans_list:
if fans_uid == item['_source']['uid']:
if fans_num_dict.has_key(str(fans_uid)):
fans_num_dict[str(fans_uid)] = fans_num_dict[
str(fans_uid)] + 1
else:
fans_num_dict[str(fans_uid)] = 1
for followers_uid in followers_list:
if followers_uid == item['_source']['uid']:
if followers_num_dict.has_key(str(followers_uid)):
fans_num_dict[str(
followers_uid
)] = fans_num_dict[str(followers_uid)] + 1
else:
fans_num_dict[str(followers_uid)] = 1
#计算影响力
origin_influence_value = (item['_source']['comment'] +
item['_source']['retweeted']) * (
1 +
item['_source']['sensitive'])
fans_value = judge_user_type(item['_source']['uid'],
fans_list)
followers_value = judge_user_type(item['_source']['uid'],
followers_list)
item['_source'][
'weibo_influence_value'] = origin_influence_value * (
fans_value + followers_value)
weibo_result.append(item['_source'])
#统计影响力、时间
event_influence_sum = event_influence_sum + item[
'_source']['weibo_influence_value']
event_time_sum = item['_source']['timestamp']
#典型微博信息
weibo_result.sort(key=lambda k:
(k.get('weibo_influence_value', 0)),
reverse=True)
event_warming_content['main_weibo_info'] = weibo_result
#事件影响力和事件时间
number = len(event_results)
event_warming_content[
'event_influence'] = event_influence_sum / number
event_warming_content['event_time'] = event_time_sum / number
else:
pass
except:
event_warming_content['main_weibo_info'] = []
event_warming_content['event_influence'] = []
event_warming_content['event_time'] = []
#print event_item[0],'event_search_time:',time.time()
try:
if event_sensitive_count > 0:
#对用户进行排序
temp_userid_dict = union_dict(fans_num_dict,
followers_num_dict)
main_userid_dict = union_dict(temp_userid_dict,
alluser_num_dict)
main_userid_dict = sorted(main_userid_dict.items(),
key=lambda d: d[1],
reverse=True)
main_userid_list = []
for i in xrange(0, len(main_userid_dict)):
main_userid_list.append(main_userid_dict[i][0])
#print 'main_userid_list:',main_userid_list
#主要参与用户信息
main_user_info = []
user_es_result = es_user_profile.mget(
index=profile_index_name,
doc_type=profile_index_type,
body={'ids': main_userid_list})['docs']
for item in user_es_result:
#print 'item:',item
#print 'found:',item['found']
#print 'id:',item['_id']
user_dict = dict()
if item['found']:
user_dict['photo_url'] = item['_source']['photo_url']
user_dict['uid'] = item['_id']
user_dict['nick_name'] = item['_source']['nick_name']
user_dict['favoritesnum'] = item['_source'][
'favoritesnum']
user_dict['fansnum'] = item['_source']['fansnum']
else:
user_dict['photo_url'] = ''
user_dict['uid'] = item['_id']
user_dict['nick_name'] = ''
user_dict['favoritesnum'] = 0
user_dict['fansnum'] = 0
main_user_info.append(user_dict)
event_warming_content['main_user_info'] = main_user_info
#print 'main_user_info:',main_user_info
#print user_es_result
'''
user_query_body={
'query':{
'filtered':{
'filter':{
'terms':{'uid':main_userid_list}
}
}
}
}
user_es_result=es_user_profile.search(index=profile_index_name,doc_type=profile_index_type,body=user_query_body)['hits']['hits']
#print user_es_result
main_user_info=[]
for item in user_es_result:
main_user_info.append(item['_source'])
event_warming_content['main_user_info']=main_user_info
'''
else:
event_warming_content['main_user_info'] = []
except:
event_warming_content['main_user_info'] = []
#print 'user_search_time:',time.time()
if event_sensitive_count > 0:
#print event_warming_content['event_name']
event_warming_list.append(event_warming_content)
else:
pass
#main_userid_list=['5536381570','2192435767','1070598590']
#user_es_result=es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,body={'ids':main_userid_list})
#print 'user_es_result',user_es_result
#print 'end_time:',time.time()
return event_warming_list
def show_personnal_warming(xnr_user_no, day_time):
#查询关注列表
try:
es_xnr_result = es_xnr.get(
index=weibo_xnr_fans_followers_index_name,
doc_type=weibo_xnr_fans_followers_index_type,
id=xnr_user_no)['_source']
followers_list = es_xnr_result['followers_list']
except:
followers_list = []
#计算敏感度排名靠前的用户
query_body = {
'query': {
'filtered': {
'filter': {
'terms': {
'uid': followers_list
}
}
}
},
'aggs': {
'followers_sensitive_num': {
'terms': {
'field': 'uid'
},
'aggs': {
'sensitive_num': {
'sum': {
'field': 'sensitive'
}
}
}
}
},
'size': MAX_VALUE
}
#测试状态下时间设置
if S_TYPE == 'test':
test_day_date = S_DATE_BCI
test_day_time = datetime2ts(test_day_date)
flow_text_index_list = get_flow_text_index_list(test_day_time)
else:
flow_text_index_list = get_flow_text_index_list(day_time)
#print flow_text_index_list
try:
first_sum_result=es_flow_text.search(index=flow_text_index_list,doc_type=flow_text_index_type,\
body=query_body)['aggregations']['followers_sensitive_num']['buckets']
except:
first_sum_result = []
#print first_sum_result
top_userlist = []
if USER_NUM < len(first_sum_result):
temp_num = USER_NUM
else:
temp_num = len(first_sum_result)
#print temp_num
for i in xrange(0, temp_num):
user_sensitive = first_sum_result[i]['sensitive_num']['value']
if user_sensitive > 0:
user_dict = dict()
user_dict['uid'] = first_sum_result[i]['key']
user_dict['sensitive'] = user_sensitive
top_userlist.append(user_dict)
else:
pass
#查询敏感用户的最敏感微博内容
results = []
for user in top_userlist:
#print user
user_detail = dict()
user_detail['uid'] = user['uid']
user_detail['user_sensitive'] = user['sensitive']
try:
user_result = es_user_profile.get(index=profile_index_name,
doc_type=profile_index_type,
id=user['uid'])['_source']
user_detail['user_name'] = user_result['nick_name']
except:
user_detail['user_name'] = ''
query_body = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'term': {
'uid': user['uid']
}
}, {
'range': {
'sensitive': {
'gte': 1,
'lte': 100
}
}
}]
}
}
}
},
'size': USER_CONTENT_NUM,
'sort': {
'sensitive': {
'order': 'desc'
}
}
}
#if S_TYPE == 'test':
try:
second_result = es_flow_text.search(
index=flow_text_index_list,
doc_type=flow_text_index_type,
body=query_body)['hits']['hits']
except:
second_result = []
#else:
# second_result=es_xnr.search(index=weibo_xnr_flow_text_listname,doc_type=xnr_flow_text_index_type,body=query_body)['hits']['hits']
s_result = []
tem_word_one = '静坐'
tem_word_two = '集合'
for item in second_result:
sensitive_words = item['_source']['sensitive_words_string']
if ((sensitive_words == tem_word_one)
or (sensitive_words == tem_word_two)):
pass
else:
s_result.append(item['_source'])
s_result.sort(key=lambda k: (k.get('sensitive', 0)), reverse=True)
user_detail['content'] = s_result
results.append(user_detail)
results.sort(key=lambda k: (k.get('user_sensitive', 0)), reverse=True)
return results
def show_speech_warming(xnr_user_no, show_type, day_time):
#关注用户
try:
es_xnr_result = es_xnr.get(
index=weibo_xnr_fans_followers_index_name,
doc_type=weibo_xnr_fans_followers_index_type,
id=xnr_user_no)['_source']
followers_list = es_xnr_result['followers_list']
except:
followers_list = []
show_condition_list = []
if show_type == 0: #全部用户
show_condition_list.append(
{'must': {
'range': {
'sensitive': {
'gte': 1,
'lte': 100
}
}
}})
elif show_type == 1: #关注用户
show_condition_list.append({
'must': [{
'terms': {
'uid': followers_list
}
}, {
'range': {
'sensitive': {
'gte': 1,
'lte': 100
}
}
}]
})
elif show_type == 2: #未关注用户
show_condition_list.append({
'must_not': {
'terms': {
'uid': followers_list
}
},
'must': {
'range': {
'sensitive': {
'gte': 1,
'lte': 100
}
}
}
})
query_body = {
'query': {
'filtered': {
'filter': {
'bool': show_condition_list[0]
}
}
},
'size': SPEECH_WARMING_NUM,
'sort': {
'sensitive': {
'order': 'desc'
}
}
}
#测试状态下时间设置
if S_TYPE == 'test':
test_day_date = S_DATE_BCI
test_day_time = datetime2ts(test_day_date)
flow_text_index_list = get_flow_text_index_list(test_day_time)
else:
flow_text_index_list = get_flow_text_index_list(day_time)
#try:
results = es_flow_text.search(index=flow_text_index_list,
doc_type=flow_text_index_type,
body=query_body)['hits']['hits']
result = []
un_id_list = [
'4045093692450438', '4045096116622444', '4045095374193153',
'4045095567336676', '4045092304116237', '4045093297982719',
'4045178576337277', '4044647661388452'
]
for item in results:
if item['_id'] in un_id_list:
pass
else:
result.append(item['_source'])
#except:
# result=[]
return result
