def lookup_todayweibo_date_warming(keywords,today_datetime):
keyword_query_list=[]
for keyword in keywords:
keyword_query_list.append({'wildcard':{'text':'*'+keyword.encode('utf-8')+'*'}})
flow_text_index_name=get_day_flow_text_index_list(today_datetime)
query_body={
'query':{
'bool':
{
'should':keyword_query_list
# 'must':{'range':{'sensitive':{'gte':1}}}
}
},
'size':MAX_WARMING_SIZE,
'sort':{'sensitive':{'order':'desc'}}
}
try:
temp_result=es_flow_text.search(index=flow_text_index_name,doc_type=flow_text_index_type,body=query_body)['hits']['hits']
date_result=[]
for item in temp_result:
item['_source']['nick_name']=get_user_nickname(item['_source']['uid'])
date_result.append(item['_source'])
except:
date_result=[]
return date_result
def get_hashtag(today_datetime):
weibo_flow_text_index_name=get_day_flow_text_index_list(today_datetime)
# query_body_test={
# 'query':{
# 'match_all':{}
# },
# 'size':999
# }
# weibo_result_test=es_flow_text.search(index=weibo_flow_text_index_name,doc_type=flow_text_index_type,body=query_body_test)['hits']['hits']
# for item in weibo_result_test:
# print 'item:',item['_source']
# if item['_source']['hashtag']:
# print 'hashtag_mark:',item['_source']['hashtag']
# else:
# pass
query_body={
'query':{
'filtered':{
'filter':{
'bool':{
'must':[
{'range':{'sensitive':{'gte':1}}}
]
}}
}
},
'aggs':{
'all_hashtag':{
'terms':{'field':'hashtag'},
'aggs':{'sum_sensitive':{
'sum':{'field':'sensitive'}
}
}
}
},
'size':5
}
weibo_text_exist=es_flow_text.search(index=weibo_flow_text_index_name,doc_type=flow_text_index_type,\
body=query_body)['aggregations']['all_hashtag']['buckets']
# print 'weibo_hashtag:',weibo_text_exist
hashtag_list = []
for item in weibo_text_exist:
event_dict=dict()
if item['key']:
# print item['key']
event_dict['event_name'] = item['key']
event_dict['event_count'] = item['doc_count']
event_dict['event_sensitive'] = item['sum_sensitive']['value']
hashtag_list.append(event_dict)
else:
pass
hashtag_list.sort(key=lambda k:(k.get('event_sensitive',0),k.get('event_count',0)),reverse=True)
# print 'hashtag_list:',hashtag_list
return hashtag_list
def lookup_today_fullkeywords(from_ts,to_ts):
query_body={
'query':{
'filtered':{
'filter':{
'bool':{
'must':[
{'range':{'timestamp':{'gte':from_ts,'lte':to_ts}}}
]
}
}
}
},
'aggs':{
'keywords':{
'terms':{
'field':'keywords_string',
'size': 80
}
}
}
}
flow_text_index_name = flow_text_index_name_pre + ts2datetime(to_ts)
try:
flow_text_exist=es_flow_text.search(index=flow_text_index_name,doc_type=xnr_flow_text_index_type,\
body=query_body)['aggregations']['keywords']['buckets']
word_dict = dict()
word_dict_new = dict()
keywords_string = ''
for item in flow_text_exist:
word = item['key']
count = item['doc_count']
word_dict[word] = count
keywords_string += '&'
keywords_string += item['key']
k_dict = extract_keywords(keywords_string)
for item_item in k_dict:
keyword = item_item.word
# print 'keyword::',keyword,type(keyword)
if word_dict.has_key(keyword):
word_dict_new[keyword] = word_dict[keyword]
else:
word_dict_new[keyword] = 1
# print 'count:',word_dict_new[keyword]
except:
word_dict_new = dict()
return word_dict_new
def lookup_today_keywords(from_ts, to_ts, xnr_user_no):
userslist = lookup_weiboxnr_concernedusers(xnr_user_no)
query_body = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'terms': {
'uid': userslist
}
}, {
'range': {
'timestamp': {
'gte': from_ts,
'lte': to_ts
}
}
}]
}
}
}
},
'aggs': {
'keywords': {
'terms': {
'field': 'keywords_string',
'size': 50
}
}
}
}
flow_text_index_name = flow_text_index_name_pre + ts2datetime(to_ts)
#print 'flow_text_index_name:', flow_text_index_name
#try:
#if S_TYPE == 'test':
# temp_flow_text_index_name='flow_text_2016-11-19'
# flow_text_exist=es_flow_text.search(index=temp_flow_text_index_name,doc_type=flow_text_index_type,\
# body=query_body)['aggregations']['keywords']['buckets']
#else:
flow_text_exist=es_flow_text.search(index=flow_text_index_name,doc_type=xnr_flow_text_index_type,\
body=query_body)['aggregations']['keywords']['buckets']
#except:
# flow_text_exist=[]
word_dict = dict()
for item in flow_text_exist:
word = item['key']
count = item['doc_count']
word_dict[word] = count
#print 'word_dict:', word_dict
return word_dict
def lookup_weibo_date_warming(keywords, today_time):
keyword_query_list = []
for keyword in keywords:
#keyword_query_list=keyword+''
keyword_query_list.append(
{'wildcard': {
'text': '*' + keyword.encode('utf-8') + '*'
}})
if S_TYPE == 'test':
test_time_gap = DAY * WARMING_DAY
end_time = datetime2ts(S_DATE_BCI)
start_time = end_time - test_time_gap
flow_text_index_name_list = get_xnr_flow_text_index_listname(
flow_text_index_name_pre, start_time, end_time)
#print flow_text_index_name_list
else:
start_time = today_time - DAY * WARMING_DAY
flow_text_index_name_list = get_xnr_flow_text_index_listname(
flow_text_index_name_pre, start_time, today_time)
#print start_time,end_time,flow_text_index_name_pre,flow_text_index_name_list
query_body = {
'query': {
'bool': {
'should': keyword_query_list
}
},
'size': SPEECH_WARMING_NUM
}
try:
temp_result = es_flow_text.search(index=flow_text_index_name_list,
doc_type=flow_text_index_type,
body=query_body)['hits']['hits']
date_result = []
#print temp_result
for item in temp_result:
date_result.append(item['_source'])
except:
date_result = []
return date_result
def lookup_today_speech_warming(xnr_user_no,show_type,start_time,end_time):
#查询关注列表
lookup_type='followers_list'
followers_list=lookup_xnr_fans_followers(xnr_user_no,lookup_type)
show_condition_list=[]
if show_type == 0: #全部用户
show_condition_list.append({'must':[{'range':{'sensitive':{'gte':1}}},{'range':{'timestamp':{'gte':start_time,'lte':end_time}}}]})
elif show_type == 1: #关注用户
show_condition_list.append({'must':[{'terms':{'uid':followers_list}},{'range':{'sensitive':{'gte':1}}},{'range':{'timestamp':{'gte':start_time,'lte':end_time}}}]})
elif show_type ==2: #未关注用户
show_condition_list.append({'must_not':{'terms':{'uid':followers_list}},'must':[{'range':{'sensitive':{'gte':1}}},{'range':{'timestamp':{'gte':start_time,'lte':end_time}}}]})
query_body={
'query':{
'filtered':{
'filter':{
'bool':show_condition_list[0]
}
}
},
'size':SPEECH_WARMING_NUM,
'sort':{'sensitive':{'order':'desc'}}
}
flow_text_index_name=get_day_flow_text_index_list(start_time)
result=[]
try:
results=es_flow_text.search(index=flow_text_index_name,doc_type=flow_text_index_type,body=query_body)['hits']['hits']
for item in results:
item['_source']['nick_name']=get_user_nickname(item['_source']['uid'])
item['_source']['_id']=item['_id']
result.append(item['_source'])
except:
result=[]
return result
def get_generate_example_model(domain_name,role_name):
domain_pinyin = pinyin.get(domain_name,format='strip',delimiter='_')
role_en = domain_ch2en_dict[role_name]
task_id = domain_pinyin + '_' + role_en
es_result = es.get(index=weibo_role_index_name,doc_type=weibo_role_index_type,id=task_id)['_source']
item = es_result
print 'es_result:::',es_result
# 政治倾向
political_side = json.loads(item['political_side'])[0][0]
if political_side == 'mid':
item['political_side'] = u'中立'
elif political_side == 'left':
item['political_side'] = u'左倾'
else:
item['political_side'] = u'右倾'
# 心理特征
psy_feature_list = []
psy_feature = json.loads(item['psy_feature'])
for i in range(TOP_PSY_FEATURE):
psy_feature_list.append(psy_feature[i][0])
item['psy_feature'] = '&'.join(psy_feature_list)
role_group_uids = json.loads(item['member_uids'])
mget_results = es_user_portrait.mget(index=portrait_index_name,doc_type=portrait_index_type,body={'ids':role_group_uids})['docs']
# topic_list = []
# for mget_item in mget_results:
# if mget_item['found']:
# keywords_list = json.loads(mget_item['_source']['keywords'])
# topic_list.extend(keywords_list)
# topic_keywords_dict = {}
# for topic_item in topic_list:
# keyword = topic_item[0]
# keyword_count = topic_item[1]
# try:
# topic_keywords_dict[keyword] += keyword_count
# except:
# topic_keywords_dict[keyword] = keyword_count
# monitor_keywords_list = []
# for i in range(3):
# keyword_max = max(topic_keywords_dict,key=topic_keywords_dict.get)
# monitor_keywords_list.append(keyword_max)
# del topic_keywords_dict[keyword_max]
# item['monitor_keywords'] = '&'.join(monitor_keywords_list)
if S_TYPE == 'test':
current_time = datetime2ts(S_DATE)
else:
current_time = int(time.time())
index_name_list = get_flow_text_index_list(current_time)
query_body_search = {
'query':{
'filtered':{
'filter':{
'terms':{'uid':role_group_uids}
}
}
},
'size':MAX_VALUE,
'_source':['keywords_string']
}
es_keyword_results = es_flow_text.search(index=index_name_list,doc_type=flow_text_index_type,\
body=query_body_search)['hits']['hits']
keywords_string = ''
for mget_item in es_keyword_results:
#print 'mget_item:::',mget_item
#if mget_item['found']:
keywords_string += '&'
keywords_string += mget_item['_source']['keywords_string']
k_dict = extract_keywords(keywords_string)
monitor_keywords_list = []
for item_item in k_dict:
monitor_keywords_list.append(item_item.word.encode('utf-8'))
item['monitor_keywords'] = ','.join(monitor_keywords_list)
mget_results_user = es_user_portrait.mget(index=profile_index_name,doc_type=profile_index_type,body={'ids':role_group_uids})['docs']
item['nick_name'] = []
for mget_item in mget_results_user:
#print 'mget_item:::',mget_item
if mget_item['found']:
item['nick_name'] = mget_item['_source']['nick_name']
item['location'] = mget_item['_source']['user_location']
item['gender'] = mget_item['_source']['sex']
uid = mget_item['_source']['uid']
try:
profile_results = es_user_portrait.get(index=profile_index_name,doc_type=profile_index_type,id=uid)['_source']
if profile_results['description']:
item['description'] = profile_results['description']
break
except:
pass
item['business_goal'] = u'渗透'
item['daily_interests'] = u'旅游'
# if S_TYPE == 'test':
# user_mget_results = es.mget(index=profile_index_name,doc_type=profile_index_type,body={'ids':role_group_uids})['docs']
# if user_mget_results
item['age'] = 30
item['career'] = u'自由职业'
active_time_list_np = np.array(json.loads(item['active_time']))
active_time_list_np_sort = np.argsort(-active_time_list_np)[:TOP_ACTIVE_TIME]
item['active_time'] = active_time_list_np_sort.tolist()
day_post_num_list = np.array(json.loads(item['day_post_num']))
item['day_post_num'] = np.mean(day_post_num_list).tolist()
item['role_name'] = role_name
task_id_new =domain_pinyin + '_' + role_en
example_model_file_name = EXAMPLE_MODEL_PATH + task_id_new + '.json'
try:
with open(example_model_file_name,"w") as dump_f:
json.dump(item,dump_f)
item_dict = dict()
#item_dict['xnr_user_no'] = xnr_user_no
item_dict['domain_name'] = domain_name
item_dict['role_name'] = role_name
es.index(index=weibo_example_model_index_name,doc_type=weibo_example_model_index_type,\
body=item_dict,id=task_id_new)
mark = True
except:
mark = False
return mark
def get_community_content(now_time, uid_list, order_by):
#step1:获取uid_list
#step2:生成话题词云、敏感词云、敏感帖子
query_body = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'terms': {
'uid': uid_list
}
}]
}
}
}
},
'aggs': {
'sensitive_words_string': {
'terms': {
'field': 'sensitive_words_string',
'size': 800
}
}
},
'size': 50,
'sort': {
order_by: {
'order': 'desc'
}
}
}
query_hashtag = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'terms': {
'uid': uid_list
}
}]
}
}
}
},
'aggs': {
'hashtag': {
'terms': {
'field': 'hashtag',
'size': 800
}
}
},
'size': 50,
'sort': {
order_by: {
'order': 'desc'
}
}
}
flow_text_index_name_list = get_flow_text_index_list(now_time)
flow_text_exist = es_flow_text.search(index = flow_text_index_name_list,doc_type = flow_text_index_type,\
body = query_body)
sensitive_keywords_result = flow_text_exist['aggregations'][
'sensitive_words_string']['buckets']
content_result = flow_text_exist['hits']['hits']
hashtag_result = es_flow_text.search(index = flow_text_index_name_list,doc_type = flow_text_index_type,\
body = query_hashtag)['aggregations']['hashtag']['buckets']
# print 'content_result:',content_result
# print 'sensitive_keywords_result:',sensitive_keywords_result
# print 'hashtag_result:',hashtag_result
sensitive_keywords_dict = get_word_count(sensitive_keywords_result)
hashtag_dict = get_word_count(hashtag_result)
content_list = []
for item in content_result:
content_list.append(item['_source'])
result_dict = dict()
result_dict['topic_wordcloud'] = hashtag_dict
result_dict['sensitive_wordcloud'] = sensitive_keywords_dict
result_dict['content_post'] = content_list
return result_dict
def create_event_warning(xnr_user_no,today_datetime,write_mark):
#获取事件名称
hashtag_list = get_hashtag(today_datetime)
#print 'hashtag_list::',hashtag_list
flow_text_index_name = get_day_flow_text_index_list(today_datetime)
#虚拟人的粉丝列表和关注列表
try:
es_xnr_result=es_xnr.get(index=weibo_xnr_fans_followers_index_name,doc_type=weibo_xnr_fans_followers_index_type,id=xnr_user_no)['_source']
followers_list=es_xnr_result['followers_list']
fans_list=es_xnr_result['fans_list']
except:
followers_list=[]
fans_list=[]
event_warming_list=[]
event_num=0
for event_item in hashtag_list:
event_sensitive_count=0
event_warming_content=dict() #事件名称、主要参与用户、典型微博、事件影响力、事件平均时间
event_warming_content['event_name']=event_item['event_name']
# print 'event_name:',event_item
event_num=event_num+1
# print 'event_num:::',event_num
# print 'first_time:::',int(time.time())
event_influence_sum=0
event_time_sum=0
query_body={
'query':{
# 'bool':{
# 'must':[{'wildcard':{'text':'*'+event_item[0]+'*'}},
# {'range':{'sensitive':{'gte':1}}}]
# }
'filtered':{
'filter':{
'bool':{
'must':[
{'term':{'hashtag':event_item['event_name']}},
{'range':{'sensitive':{'gte':1}}}
]
}
}
}
},
'size':MAX_WARMING_SIZE,
'sort':{'sensitive':{'order':'desc'}}
}
#try:
event_results=es_flow_text.search(index=flow_text_index_name,doc_type=flow_text_index_type,body=query_body)['hits']['hits']
if event_results:
weibo_result=[]
fans_num_dict=dict()
followers_num_dict=dict()
alluser_num_dict=dict()
# print 'sencond_time:::',int(time.time())
for item in event_results:
#print 'event_content:',item['_source']['text']
#统计用户信息
if alluser_num_dict.has_key(str(item['_source']['uid'])):
followers_mark=set_intersection(item['_source']['uid'],followers_list)
if followers_mark > 0:
alluser_num_dict[str(item['_source']['uid'])]=alluser_num_dict[str(item['_source']['uid'])]+1*2
else:
alluser_num_dict[str(item['_source']['uid'])]=alluser_num_dict[str(item['_source']['uid'])]+1
else:
alluser_num_dict[str(item['_source']['uid'])]=1
#计算影响力
origin_influence_value=(1+item['_source']['comment']+item['_source']['retweeted'])*(1+item['_source']['sensitive'])
# fans_value=judge_user_type(item['_source']['uid'],fans_list)
followers_value=judge_user_type(item['_source']['uid'],followers_list)
item['_source']['weibo_influence_value']=origin_influence_value*(followers_value)
item['_source']['nick_name']=get_user_nickname(item['_source']['uid'])
weibo_result.append(item['_source'])
#统计影响力、时间
event_influence_sum=event_influence_sum+item['_source']['weibo_influence_value']
event_time_sum=event_time_sum+item['_source']['timestamp']
# print 'third_time:::',int(time.time())
#典型微博信息
weibo_result.sort(key=lambda k:(k.get('weibo_influence_value',0)),reverse=True)
event_warming_content['main_weibo_info']=json.dumps(weibo_result)
#事件影响力和事件时间
number=len(event_results)
event_warming_content['event_influence']=event_influence_sum/number
event_warming_content['event_time']=event_time_sum/number
# except:
# event_warming_content['main_weibo_info']=[]
# event_warming_content['event_influence']=0
# event_warming_content['event_time']=0
# try:
#对用户进行排序
alluser_num_dict=sorted(alluser_num_dict.items(),key=lambda d:d[1],reverse=True)
main_userid_list=[]
for i in xrange(0,len(alluser_num_dict)):
main_userid_list.append(alluser_num_dict[i][0])
#主要参与用户信息
main_user_info=[]
user_es_result=es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,body={'ids':main_userid_list})['docs']
for item in user_es_result:
user_dict=dict()
if item['found']:
user_dict['photo_url']=item['_source']['photo_url']
user_dict['uid']=item['_id']
user_dict['nick_name']=item['_source']['nick_name']
user_dict['favoritesnum']=item['_source']['favoritesnum']
user_dict['fansnum']=item['_source']['fansnum']
else:
user_dict['photo_url']=''
user_dict['uid']=item['_id']
user_dict['nick_name']=''
user_dict['favoritesnum']=0
user_dict['fansnum']=0
main_user_info.append(user_dict)
event_warming_content['main_user_info']=json.dumps(main_user_info)
# except:
# event_warming_content['main_user_info']=[]
# print 'fourth_time:::',int(time.time())
event_warming_content['xnr_user_no']=xnr_user_no
event_warming_content['validity']=0
event_warming_content['timestamp']=today_datetime
now_time=int(time.time())
# task_id=xnr_user_no+'_'+str(now_time)
event_warming_content['_id']=xnr_user_no+'_'+event_warming_content['event_name']
task_id=event_warming_content['_id']
if write_mark:
# print 'today_datetime:::',ts2datetime(today_datetime)
mark=write_envent_warming(today_datetime,event_warming_content,task_id)
event_warming_list.append(mark)
else:
event_warming_list.append(event_warming_content)
else:
pass
# print 'fifth_time:::',int(time.time())
return event_warming_list
def lookup_today_personal_warming(xnr_user_no,start_time,end_time):
#查询关注列表
lookup_type='followers_list'
followers_list=lookup_xnr_fans_followers(xnr_user_no,lookup_type)
#查询虚拟人uid
xnr_uid=lookup_xnr_uid(xnr_user_no)
#计算敏感度排名靠前的用户
query_body={
'query':{
'filtered':{
'filter':{
'bool':{
'must':[
# {'terms':{'uid':followers_list}},
{'range':{
'timestamp':{
'gte':start_time,
'lte':end_time
}
}}
]
}
}
}
},
'aggs':{
'followers_sensitive_num':{
'terms':{'field':'uid'},
'aggs':{
'sensitive_num':{
'sum':{'field':'sensitive'}
}
}
}
},
'size':MAX_SEARCH_SIZE
}
flow_text_index_name=get_day_flow_text_index_list(end_time)
try:
first_sum_result=es_flow_text.search(index=flow_text_index_name,doc_type=flow_text_index_type,\
body=query_body)['aggregations']['followers_sensitive_num']['buckets']
except:
first_sum_result=[]
#print first_sum_result
top_userlist=[]
for i in xrange(0,len(first_sum_result)):
user_sensitive=first_sum_result[i]['sensitive_num']['value']
if user_sensitive > 0:
user_dict=dict()
user_dict['uid']=first_sum_result[i]['key']
user_dict['sensitive']=user_sensitive
top_userlist.append(user_dict)
else:
pass
#查询敏感用户的敏感微博内容
results=[]
for user in top_userlist:
#print user
user_detail=dict()
user_detail['uid']=user['uid']
user_detail['user_sensitive']=user['sensitive']
user_lookup_id=xnr_uid+'_'+user['uid']
print user_lookup_id
try:
#user_result=es_xnr.get(index=weibo_feedback_follow_index_name,doc_type=weibo_feedback_follow_index_type,id=user_lookup_id)['_source']
user_result=es_user_profile.get(index=profile_index_name,doc_type=profile_index_type,id=user['uid'])['_source']
user_detail['user_name']=user_result['nick_name']
except:
user_detail['user_name']=''
query_body={
'query':{
'filtered':{
'filter':{
'bool':{
'must':[
{'term':{'uid':user['uid']}},
{'range':{'sensitive':{'gte':1}}}
]
}
}
}
},
'size':MAX_WARMING_SIZE,
'sort':{'sensitive':{'order':'desc'}}
}
try:
second_result=es_flow_text.search(index=flow_text_index_name,doc_type=flow_text_index_type,body=query_body)['hits']['hits']
except:
second_result=[]
s_result=[]
for item in second_result:
item['_source']['nick_name']=get_user_nickname(item['_source']['uid'])
s_result.append(item['_source'])
s_result.sort(key=lambda k:(k.get('sensitive',0)),reverse=True)
user_detail['content']=json.dumps(s_result)
user_detail['xnr_user_no']=xnr_user_no
user_detail['validity']=0
user_detail['timestamp']=end_time
user_detail['_id']=xnr_user_no+'_'+user_detail['uid']
results.append(user_detail)
results.sort(key=lambda k:(k.get('user_sensitive',0)),reverse=True)
return results
def get_recommend_follows(task_detail):
recommend_results = dict()
daily_interests_list = task_detail['daily_interests'].encode('utf-8').split('，')
monitor_keywords_list = task_detail['monitor_keywords'].encode('utf-8').split('，')
#print 'daily_interests_list::',daily_interests_list
create_time = time.time()
if S_TYPE == 'test':
create_time = datetime2ts(S_DATE)
index_name_list = get_flow_text_index_list(create_time)
## 日常兴趣关注
try:
query_body = {
'query':{
'filtered':{
'filter':{
'terms':{'daily_interests':daily_interests_list}
}
}
},
'sort':{'user_fansnum':{'order':'desc'}},
'size':DAILY_INTEREST_TOP_USER,
'_source':['uid']
}
es_results = es_flow_text.search(index=index_name_list,doc_type='text',body=query_body)['hits']['hits']
daily_interest_uid_set = set()
for result in es_results:
daily_interest_uid_set.add(result['_source']['uid'])
daily_interest_uid_list = list(daily_interest_uid_set)
es_daily_interests_results = es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,\
body={'ids':daily_interest_uid_list})['docs']
nick_name_dict = {}
es_daily_interests_results = es_daily_interests_results[:max(NICK_NAME_TOP,len(es_daily_interests_results))]
for result in es_daily_interests_results:
if result['found'] == True:
result = result['_source']
nick_name_dict[result['uid']] = result['nick_name']
else:
continue
recommend_results['daily_interests'] = nick_name_dict
except:
print '没有找到日常兴趣相符的用户'
recommend_results['daily_interests'] = {}
## 监测词关注
nest_query_list = []
#print 'monitor_keywords_list:::',monitor_keywords_list
for monitor_keyword in monitor_keywords_list:
nest_query_list.append({'wildcard':{'keywords_string':'*'+monitor_keyword+'*'}})
#print 'nest_query_list::',nest_query_list
try:
query_body_monitor = {
'query':{
'bool':{
'must':nest_query_list
}
},
'sort':{'user_fansnum':{'order':'desc'}},
'size':MONITOR_TOP_USER,
'_source':['uid']
}
#print '123'
es_results = es_flow_text.search(index=index_name_list,doc_type='text',body=query_body_monitor)['hits']['hits']
#print 'es_results::',es_results
monitor_keywords_uid_set = set()
for result in es_results:
monitor_keywords_uid_set.add(result['_source']['uid'])
monitor_keywords_uid_list = list(monitor_keywords_uid_set)
es_monitor_keywords_results = es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,\
body={'ids':monitor_keywords_uid_list})['docs']
nick_name_dict = {}
es_monitor_keywords_results = es_monitor_keywords_results[:max(NICK_NAME_TOP,len(es_monitor_keywords_results))]
for result in es_monitor_keywords_results:
if result['found'] == True:
result = result['_source']
nick_name_dict[result['uid']] = result['nick_name']
else:
continue
recommend_results['monitor_keywords'] = nick_name_dict
except:
print '没有找到监测词相符的用户'
recommend_results['monitor_keywords'] = {}
# print 'recommend_results::',recommend_results
return recommend_results
def lookup_hot_posts(from_ts, to_ts, weiboxnr_id, classify_id, order_id):
#step 1 :adjust the time condition for time
time_gap = to_ts - from_ts
now_time = time.time()
test_time_gap = datetime2ts(ts2datetime(now_time)) - datetime2ts(S_DATE)
#print 'from, to:', from_ts, to_ts
if S_TYPE == 'test':
today_date_time = datetime2ts(S_DATE)
from_ts = from_ts - test_time_gap
#to_ts = to_ts - test_time_gap
to_ts = from_ts + MAX_FLOW_TEXT_DAYS * DAY
from_date_ts = datetime2ts(ts2datetime(from_ts))
to_date_ts = datetime2ts(ts2datetime(to_ts))
#print 'from_date_ts, to_date_ts:', ts2date(from_date_ts), ts2date(to_date_ts)
#print from_date_ts,to_date_ts
flow_text_index_name_list = []
days_num = MAX_FLOW_TEXT_DAYS
for i in range(0, (days_num + 1)):
date_range_start_ts = to_date_ts - i * DAY
date_range_start_datetime = ts2datetime(date_range_start_ts)
index_name = flow_text_index_name_pre + date_range_start_datetime
if es_flow_text.indices.exists(index=index_name):
flow_text_index_name_list.append(index_name)
else:
pass
if order_id == 1: #按时间排序
sort_condition_list = [{'timestamp': {'order': 'desc'}}]
elif order_id == 2: #按热度排序
sort_condition_list = [{'retweeted': {'order': 'desc'}}]
elif order_id == 3: #按敏感度排序
sort_condition_list = [{'sensitive': {'order': 'desc'}}]
#else: #默认设为按时间排序
# sort_condition_list=[{'timestamp':{'order':'desc'}}]
userslist = lookup_weiboxnr_concernedusers(weiboxnr_id)
#全部用户 0，已关注用户 1，未关注用户-1
range_time_list = {
'range': {
'timestamp': {
'gte': int(from_ts),
'lt': int(to_ts)
}
}
}
print range_time_list
user_condition_list = []
if classify_id == 1:
user_condition_list = [{
'bool': {
'must': [{
'terms': {
'uid': userslist
}
}, range_time_list]
}
}]
elif classify_id == 2:
user_condition_list = [{
'bool': {
'must': [range_time_list],
'must_not': [{
'terms': {
'uid': userslist
}
}]
}
}]
elif classify_id == 0:
user_condition_list = [{'bool': {'must': [range_time_list]}}]
#print 'sort_condition_list',sort_condition_list
#print 'user_condition_list',user_condition_list
query_body = {
'query': {
'filtered': {
'filter': user_condition_list
}
},
'size': HOT_WEIBO_NUM,
'sort': sort_condition_list
}
try:
es_result=es_flow_text.search(index=flow_text_index_name_list,doc_type=flow_text_index_type,\
body=query_body)['hits']['hits']
hot_result = []
for item in es_result:
item['_source']['nick_name'] = get_user_nickname(
item['_source']['uid'])
hot_result.append(item['_source'])
except:
hot_result = []
#print 'hot_result:', hot_result
return hot_result
def show_event_warming(xnr_user_no):
now_time = int(time.time())
#print 'first_time:',time.time()
hashtag_list = get_hashtag()
#print 'hashtag_list_time:',time.time()
#print 'hashtag_list:::::::',hashtag_list
if S_TYPE == 'test':
test_day_date = S_DATE_EVENT_WARMING
test_day_time = datetime2ts(test_day_date)
flow_text_index_list = get_flow_text_index_list(test_day_time)
#print flow_text_index_list
hashtag_list = [('网络义勇军发布', 13), ('美国', 7), ('德国', 5), ('中国', 4),
('清真食品', 3), ('反邪动态', 2), ('台海观察', 2), ('雷哥微评', 2),
('中国军队', 1)]
#hashtag_list=[('网络义勇军发布',13),('美国',7),('芒果TV',6),('德国',5),('中国',4),('清真食品',3),('反邪动态',2),('台海观察',2),('每日一药',2),('雷哥微评',2),('PPAP洗脑神曲',1),('中国军队',1)]
#weibo_xnr_flow_text_listname=['flow_text_2016-11-26','flow_text_2016-11-25','flow_text_2016-11-24']
else:
flow_text_index_list = get_flow_text_index_list(now_time)
#weibo_xnr_flow_text_listname=get_xnr_flow_text_index_list(now_time)
#print flow_text_index_list,hashtag_list
#虚拟人的粉丝列表和关注列表
try:
es_xnr_result = es_xnr.get(
index=weibo_xnr_fans_followers_index_name,
doc_type=weibo_xnr_fans_followers_index_type,
id=xnr_user_no)['_source']
followers_list = es_xnr_result['followers_list']
fans_list = es_xnr_result['fans_list']
except:
followers_list = []
fans_list = []
#print 'weibo_xnr_fans_followers_time:',time.time()
event_warming_list = []
for event_item in hashtag_list:
#print event_item,event_item[0]
event_sensitive_count = 0
event_warming_content = dict() #事件名称、主要参与用户、典型微博、事件影响力、事件平均时间
event_warming_content['event_name'] = event_item[0]
event_influence_sum = 0
event_time_sum = 0
query_body = {
'query': {
'bool': {
'should': {
'wildcard': {
'text': '*' + event_item[0] + '*'
}
}
}
}
}
try:
event_results = es_flow_text.search(
index=flow_text_index_list,
doc_type=flow_text_index_type,
body=query_body)['hits']['hits']
weibo_result = []
fans_num_dict = dict()
followers_num_dict = dict()
alluser_num_dict = dict()
#print event_results
for item in event_results:
if item['_source']['sensitive'] > 0:
event_sensitive_count = event_sensitive_count + 1
#统计用户信息
if alluser_num_dict.has_key(str(item['_source']['uid'])):
alluser_num_dict[str(
item['_source']['uid'])] = alluser_num_dict[str(
item['_source']['uid'])] + 1
else:
alluser_num_dict[str(item['_source']['uid'])] = 1
for fans_uid in fans_list:
if fans_uid == item['_source']['uid']:
if fans_num_dict.has_key(str(fans_uid)):
fans_num_dict[str(fans_uid)] = fans_num_dict[
str(fans_uid)] + 1
else:
fans_num_dict[str(fans_uid)] = 1
for followers_uid in followers_list:
if followers_uid == item['_source']['uid']:
if followers_num_dict.has_key(str(followers_uid)):
fans_num_dict[str(
followers_uid
)] = fans_num_dict[str(followers_uid)] + 1
else:
fans_num_dict[str(followers_uid)] = 1
#计算影响力
origin_influence_value = (item['_source']['comment'] +
item['_source']['retweeted']) * (
1 +
item['_source']['sensitive'])
fans_value = judge_user_type(item['_source']['uid'],
fans_list)
followers_value = judge_user_type(item['_source']['uid'],
followers_list)
item['_source'][
'weibo_influence_value'] = origin_influence_value * (
fans_value + followers_value)
weibo_result.append(item['_source'])
#统计影响力、时间
event_influence_sum = event_influence_sum + item[
'_source']['weibo_influence_value']
event_time_sum = item['_source']['timestamp']
#典型微博信息
weibo_result.sort(key=lambda k:
(k.get('weibo_influence_value', 0)),
reverse=True)
event_warming_content['main_weibo_info'] = weibo_result
#事件影响力和事件时间
number = len(event_results)
event_warming_content[
'event_influence'] = event_influence_sum / number
event_warming_content['event_time'] = event_time_sum / number
else:
pass
except:
event_warming_content['main_weibo_info'] = []
event_warming_content['event_influence'] = []
event_warming_content['event_time'] = []
#print event_item[0],'event_search_time:',time.time()
try:
if event_sensitive_count > 0:
#对用户进行排序
temp_userid_dict = union_dict(fans_num_dict,
followers_num_dict)
main_userid_dict = union_dict(temp_userid_dict,
alluser_num_dict)
main_userid_dict = sorted(main_userid_dict.items(),
key=lambda d: d[1],
reverse=True)
main_userid_list = []
for i in xrange(0, len(main_userid_dict)):
main_userid_list.append(main_userid_dict[i][0])
#print 'main_userid_list:',main_userid_list
#主要参与用户信息
main_user_info = []
user_es_result = es_user_profile.mget(
index=profile_index_name,
doc_type=profile_index_type,
body={'ids': main_userid_list})['docs']
for item in user_es_result:
#print 'item:',item
#print 'found:',item['found']
#print 'id:',item['_id']
user_dict = dict()
if item['found']:
user_dict['photo_url'] = item['_source']['photo_url']
user_dict['uid'] = item['_id']
user_dict['nick_name'] = item['_source']['nick_name']
user_dict['favoritesnum'] = item['_source'][
'favoritesnum']
user_dict['fansnum'] = item['_source']['fansnum']
else:
user_dict['photo_url'] = ''
user_dict['uid'] = item['_id']
user_dict['nick_name'] = ''
user_dict['favoritesnum'] = 0
user_dict['fansnum'] = 0
main_user_info.append(user_dict)
event_warming_content['main_user_info'] = main_user_info
#print 'main_user_info:',main_user_info
#print user_es_result
'''
user_query_body={
'query':{
'filtered':{
'filter':{
'terms':{'uid':main_userid_list}
}
}
}
}
user_es_result=es_user_profile.search(index=profile_index_name,doc_type=profile_index_type,body=user_query_body)['hits']['hits']
#print user_es_result
main_user_info=[]
for item in user_es_result:
main_user_info.append(item['_source'])
event_warming_content['main_user_info']=main_user_info
'''
else:
event_warming_content['main_user_info'] = []
except:
event_warming_content['main_user_info'] = []
#print 'user_search_time:',time.time()
if event_sensitive_count > 0:
#print event_warming_content['event_name']
event_warming_list.append(event_warming_content)
else:
pass
#main_userid_list=['5536381570','2192435767','1070598590']
#user_es_result=es_user_profile.mget(index=profile_index_name,doc_type=profile_index_type,body={'ids':main_userid_list})
#print 'user_es_result',user_es_result
#print 'end_time:',time.time()
return event_warming_list
def show_personnal_warming(xnr_user_no, day_time):
#查询关注列表
try:
es_xnr_result = es_xnr.get(
index=weibo_xnr_fans_followers_index_name,
doc_type=weibo_xnr_fans_followers_index_type,
id=xnr_user_no)['_source']
followers_list = es_xnr_result['followers_list']
except:
followers_list = []
#计算敏感度排名靠前的用户
query_body = {
'query': {
'filtered': {
'filter': {
'terms': {
'uid': followers_list
}
}
}
},
'aggs': {
'followers_sensitive_num': {
'terms': {
'field': 'uid'
},
'aggs': {
'sensitive_num': {
'sum': {
'field': 'sensitive'
}
}
}
}
},
'size': MAX_VALUE
}
#测试状态下时间设置
if S_TYPE == 'test':
test_day_date = S_DATE_BCI
test_day_time = datetime2ts(test_day_date)
flow_text_index_list = get_flow_text_index_list(test_day_time)
else:
flow_text_index_list = get_flow_text_index_list(day_time)
#print flow_text_index_list
try:
first_sum_result=es_flow_text.search(index=flow_text_index_list,doc_type=flow_text_index_type,\
body=query_body)['aggregations']['followers_sensitive_num']['buckets']
except:
first_sum_result = []
#print first_sum_result
top_userlist = []
if USER_NUM < len(first_sum_result):
temp_num = USER_NUM
else:
temp_num = len(first_sum_result)
#print temp_num
for i in xrange(0, temp_num):
user_sensitive = first_sum_result[i]['sensitive_num']['value']
if user_sensitive > 0:
user_dict = dict()
user_dict['uid'] = first_sum_result[i]['key']
user_dict['sensitive'] = user_sensitive
top_userlist.append(user_dict)
else:
pass
#查询敏感用户的最敏感微博内容
results = []
for user in top_userlist:
#print user
user_detail = dict()
user_detail['uid'] = user['uid']
user_detail['user_sensitive'] = user['sensitive']
try:
user_result = es_user_profile.get(index=profile_index_name,
doc_type=profile_index_type,
id=user['uid'])['_source']
user_detail['user_name'] = user_result['nick_name']
except:
user_detail['user_name'] = ''
query_body = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'term': {
'uid': user['uid']
}
}, {
'range': {
'sensitive': {
'gte': 1,
'lte': 100
}
}
}]
}
}
}
},
'size': USER_CONTENT_NUM,
'sort': {
'sensitive': {
'order': 'desc'
}
}
}
#if S_TYPE == 'test':
try:
second_result = es_flow_text.search(
index=flow_text_index_list,
doc_type=flow_text_index_type,
body=query_body)['hits']['hits']
except:
second_result = []
#else:
# second_result=es_xnr.search(index=weibo_xnr_flow_text_listname,doc_type=xnr_flow_text_index_type,body=query_body)['hits']['hits']
s_result = []
tem_word_one = '静坐'
tem_word_two = '集合'
for item in second_result:
sensitive_words = item['_source']['sensitive_words_string']
if ((sensitive_words == tem_word_one)
or (sensitive_words == tem_word_two)):
pass
else:
s_result.append(item['_source'])
s_result.sort(key=lambda k: (k.get('sensitive', 0)), reverse=True)
user_detail['content'] = s_result
results.append(user_detail)
results.sort(key=lambda k: (k.get('user_sensitive', 0)), reverse=True)
return results
def show_speech_warming(xnr_user_no, show_type, day_time):
#关注用户
try:
es_xnr_result = es_xnr.get(
index=weibo_xnr_fans_followers_index_name,
doc_type=weibo_xnr_fans_followers_index_type,
id=xnr_user_no)['_source']
followers_list = es_xnr_result['followers_list']
except:
followers_list = []
show_condition_list = []
if show_type == 0: #全部用户
show_condition_list.append(
{'must': {
'range': {
'sensitive': {
'gte': 1,
'lte': 100
}
}
}})
elif show_type == 1: #关注用户
show_condition_list.append({
'must': [{
'terms': {
'uid': followers_list
}
}, {
'range': {
'sensitive': {
'gte': 1,
'lte': 100
}
}
}]
})
elif show_type == 2: #未关注用户
show_condition_list.append({
'must_not': {
'terms': {
'uid': followers_list
}
},
'must': {
'range': {
'sensitive': {
'gte': 1,
'lte': 100
}
}
}
})
query_body = {
'query': {
'filtered': {
'filter': {
'bool': show_condition_list[0]
}
}
},
'size': SPEECH_WARMING_NUM,
'sort': {
'sensitive': {
'order': 'desc'
}
}
}
#测试状态下时间设置
if S_TYPE == 'test':
test_day_date = S_DATE_BCI
test_day_time = datetime2ts(test_day_date)
flow_text_index_list = get_flow_text_index_list(test_day_time)
else:
flow_text_index_list = get_flow_text_index_list(day_time)
#try:
results = es_flow_text.search(index=flow_text_index_list,
doc_type=flow_text_index_type,
body=query_body)['hits']['hits']
result = []
un_id_list = [
'4045093692450438', '4045096116622444', '4045095374193153',
'4045095567336676', '4045092304116237', '4045093297982719',
'4045178576337277', '4044647661388452'
]
for item in results:
if item['_id'] in un_id_list:
pass
else:
result.append(item['_source'])
#except:
# result=[]
return result
def lookup_active_weibouser(classify_id,weiboxnr_id,start_time,end_time):
time_gap = end_time - start_time
now_time = time.time()
test_time_gap = datetime2ts(ts2datetime(now_time)) - datetime2ts(S_DATE_BCI)
#print 'from, to:', ts2date(start_time), ts2date(end_time)
if S_TYPE == 'test':
today_date_time = datetime2ts(S_DATE_BCI)
start_time = start_time - test_time_gap
end_time = end_time - test_time_gap
from_date_ts=datetime2ts(ts2datetime(start_time))
to_date_ts=datetime2ts(ts2datetime(end_time))
#print 's_date_bci:', S_DATE_BCI
#print 'from_date_ts, to_date_ts:', ts2date(from_date_ts), ts2date(to_date_ts)
bci_index_name = weibo_bci_index_name_pre + ''.join(ts2datetime(today_date_time).split('-'))
#print 'bci_index_name:', bci_index_name
#print 'end_time:', ts2date(end_time)
#step1: users condition
#make sure the users range by classify choice
userlist = lookup_weiboxnr_concernedusers(weiboxnr_id)
if classify_id == 1: #concrenedusers
condition_list=[{'bool':{'must':{'terms':{'uid':userlist}}}}]
elif classify_id == 2: #unconcrenedusers
condition_list=[{'bool':{'must_not':[{'terms':{'uid':userlist}}]}}]
elif classify_id == 0:
condition_list=[{'match_all':{}}]
print userlist,classify_id,condition_list
#step 2:lookup users
user_max_index=count_maxweibouser_influence(end_time)
results = []
for item in condition_list:
query_body={
'query':item,
'size':HOT_WEIBO_NUM, #查询影响力排名前50的用户即可
'sort':{'user_index':{'order':'desc'}}
}
try:
#print 'query_body:', query_body
flow_text_exist=es_flow_text.search(index=bci_index_name,\
doc_type=weibo_bci_index_type,body=query_body)['hits']['hits']
search_uid_list = [item['_source']['user'] for item in flow_text_exist]
weibo_user_exist = es_user_profile.search(index=profile_index_name,\
doc_type=profile_index_type,body={'query':{'terms':{'uid':search_uid_list}}})['hits']['hits']
#print 'weibo_user_exist:', weibo_user_exist
weibo_user_dict = dict()
for item in weibo_user_exist:
uid = item['_source']['uid']
weibo_user_dict[uid] = item['_source']
for item in flow_text_exist:
#print 'item:', item['_source']
influence = item['_source']['user_index']/user_max_index*100
fans_num = item['_source']['user_fansnum']
friends_num = item['_source']['user_friendsnum']
total_number = item['_source']['total_number']
uid = item['_source']['user']
try:
weibo_user_info = weibo_user_dict[uid]
uname = weibo_user_info['nick_name']
location = weibo_user_info['user_location']
url = weibo_user_info['photo_url']
except:
uname = ''
location = ''
url = ''
#print 'uid:', uid
results.append({'uid':uid, 'influence':influence, 'fans_num':fans_num, \
'total_number':total_number, 'friends_num':friends_num,\
'uname': uname, 'location':location, 'url': url})
#print 'results:', results
'''
uid=item['_source']['uid']
#微博数
item['_source']['weibos_sum']=count_weibouser_weibosum(uid,end_time)
#影响力
user_index=count_weibouser_index(uid,end_time)
if user_max_index >0:
item['_source']['influence']=user_index/user_max_index*100
else:
item['_source']['influence']=0
if item['_source']['influence']>=INFLUENCE_MIN:
results.append(item['_source'])
'''
except:
results=[]
return results
