def count_weibouser_index(uid,end_time):
try:
if S_TYPE == 'test':
temp_index_name='bci_20161121'
user_result=es_user_profile.get(index=temp_index_name,doc_type=weibo_bci_index_type,id=uid)['_source']
else:
user_result=es_user_profile.get(index=index_name,doc_type=weibo_bci_index_type,id=uid)['_source']
user_index=user_result['user_index']
except:
user_index=0
return user_index
def get_user_nickname(uid):
try:
user_result=es_user_profile.get(index=profile_index_name,doc_type=profile_index_type,id=uid)['_source']
user_name=user_result['nick_name']
except:
user_name=''
return user_name
def get_user_detail(uid):
user_result=es_user_profile.get(index=profile_index_name,doc_type=profile_index_type,id=uid)['_source']
user_detail = dict()
if user_result:
user_detail['user_url'] = "http://weibo.com/" + uid
user_detail['uid'] = uid
user_detail['nick_name'] = user_result['nick_name']
user_detail['photo_url'] = user_result['photo_url']
user_detail['user_location'] = user_result['user_location']
user_detail['sex'] = user_result['sex']
user_detail['fansnum'] = user_result['fansnum']
user_detail['friendsnum'] = user_result['friendsnum']
else:
user_detail['user_url'] = "http://weibo.com/" + uid
user_detail['uid'] = uid
user_detail['nick_name'] = ''
user_detail['photo_url'] = ''
user_detail['user_location'] = ''
user_detail['sex'] = 0
user_detail['fansnum'] = 0
user_detail['friendsnum'] = 0
return user_detail
def report_warming_content(task_detail):
report_dict=dict()
report_dict['report_type']=task_detail['report_type']
report_dict['report_time']=task_detail['report_time']
report_dict['xnr_user_no']=task_detail['xnr_user_no']
report_dict['event_name']=task_detail['event_name']
report_dict['uid']=task_detail['uid']
report_dict['nick_name']=get_user_nickname(task_detail['uid'])
weibo_list=[]
user_list=[]
# print 'type:',type(task_detail['weibo_info']),task_detail['weibo_info']
weibo_info=task_detail['weibo_info']
for item in weibo_info:
item['timestamp'] = int(item['timestamp'])
flow_text_index_name = flow_text_index_name_pre + ts2datetime(item['timestamp'])
try:
weibo_result=es_flow_text.get(index=flow_text_index_name,doc_type=flow_text_index_type,id=item['mid'])['_source']
weibo_result['nick_name']=get_user_nickname(weibo_result['uid'])
weibo_list.append(weibo_result)
except:
if task_detail['report_type']==u'人物':
weibo_user_warning_index_name = weibo_user_warning_index_name_pre + ts2datetime(item['timestamp'])
weibo_user_warming_id=task_detail['xnr_user_no']+'_'+task_detail['uid']
try:
weibo_user_result=es_xnr.get(index=weibo_user_warning_index_name,doc_type=weibo_user_warning_index_type,id=weibo_user_warming_id)['_source']
user_warning_content=json.dumps(weibo_user_result['content'])
for content in user_warning_content:
if content['mid'] == item['mid']:
weibo_list.append(content)
else:
pass
except:
print 'user_error!'
elif task_detail['report_type']==u'言论':
weibo_speech_warning_index_name = weibo_speech_warning_index_name_pre + ts2datetime(item['timestamp'])
try:
weibo_speech_result=es_xnr.get(index=weibo_speech_warning_index_name,doc_type=weibo_speech_warning_index_type,id=task_detail['xnr_user_no']+'_'+item['mid'])['_source']
report_dict['uid']=weibo_speech_result['uid']
weibo_list.append(weibo_speech_result)
except:
# weibo_timing_warning_index_name = weibo_timing_warning_index_name_pre + ts2datetime(item['timestamp'])
print 'speech_error!'
elif task_detail['report_type']==u'事件':
weibo_event_warning_index_name = weibo_event_warning_index_name_pre + ts2datetime(item['timestamp'])
event_warning_id = task_detail['xnr_user_no']+'_'+task_detail['event_name']
try:
event_result=es_xnr.get(index=weibo_event_warning_index_name,doc_type=weibo_event_warning_index_type,id=event_warning_id)['_source']
event_content=json.dumps(event_result['main_weibo_info'])
for event in event_content:
if event['mid'] == item['mid']:
weibo_list.append(event)
else:
pass
except:
print 'event_error!'
elif task_detail['report_type']==u'时间':
year = ts2yeartime(item['timestamp'])
weibo_timing_warning_index_name = weibo_timing_warning_index_name_pre + year +'_' + task_detail['date_time']
try:
time_result=es_xnr.search(index=weibo_timing_warning_index_name,doc_type=weibo_timing_warning_index_type,query_body={'query':{'match_all':{}}})['hits']['hits']
time_content=[]
for timedata in time_result:
for data in timedata['weibo_date_warming_content']:
if data['mid'] == item['mid']:
weibo_list.append(data)
else:
pass
except:
print 'time_error!'
user_info=task_detail['user_info']
if user_info:
for uid in user_info:
user=dict()
try:
user_result=es_user_profile.get(index=profile_index_name,doc_type=profile_index_type,id=uid)['_source']
user['uid']=uid
user['nick_name']=user_result['nick_name']
user['fansnum']=user_result['fansnum']
user['friendsnum']=user_result['friendsnum']
user['photo_url']=user_result['photo_url']
user_list.append(user)
except:
user['uid']=uid
user['nick_name']=''
user['fansnum']=0
user['friendsnum']=0
user['photo_url']=''
user_list.append(user)
print 'user_list error!'
else:
pass
report_content=dict()
report_content['user_list']=user_list
report_content['weibo_list']=weibo_list
report_dict['report_content']=json.dumps(report_content)
report_id=''
if task_detail['report_type'] == u'言论':
report_id=weibo_info[0]['mid']
elif task_detail['report_type'] == u'人物':
report_id=task_detail['xnr_user_no']+'_'+task_detail['uid']
elif task_detail['report_type'] == u'事件':
report_id=task_detail['xnr_user_no']+'_'+task_detail['event_name']
elif task_detail['report_type'] == u'时间':
# print weibo_info
if weibo_info:
report_id=weibo_info[0]['mid']
else:
report_id=str(task_detail['report_time'])
if weibo_list:
report_mark=True
else:
report_mark=False
#预警上报后不再显示问题
now_time=int(time.time())
weibo_report_management_index_name = weibo_report_management_index_name_pre + ts2datetime(now_time)
if es_xnr.indices.exists(index=weibo_report_management_index_name):
pass
else:
weibo_report_management_mappings()
if report_id and report_mark:
try:
es_xnr.index(index=weibo_report_management_index_name,doc_type=weibo_report_management_index_type,id=report_id,body=report_dict)
mark=True
except:
mark=False
else:
mark=False
return mark
def lookup_today_personal_warming(xnr_user_no,start_time,end_time):
#查询关注列表
lookup_type='followers_list'
followers_list=lookup_xnr_fans_followers(xnr_user_no,lookup_type)
#查询虚拟人uid
xnr_uid=lookup_xnr_uid(xnr_user_no)
#计算敏感度排名靠前的用户
query_body={
'query':{
'filtered':{
'filter':{
'bool':{
'must':[
# {'terms':{'uid':followers_list}},
{'range':{
'timestamp':{
'gte':start_time,
'lte':end_time
}
}}
]
}
}
}
},
'aggs':{
'followers_sensitive_num':{
'terms':{'field':'uid'},
'aggs':{
'sensitive_num':{
'sum':{'field':'sensitive'}
}
}
}
},
'size':MAX_SEARCH_SIZE
}
flow_text_index_name=get_day_flow_text_index_list(end_time)
try:
first_sum_result=es_flow_text.search(index=flow_text_index_name,doc_type=flow_text_index_type,\
body=query_body)['aggregations']['followers_sensitive_num']['buckets']
except:
first_sum_result=[]
#print first_sum_result
top_userlist=[]
for i in xrange(0,len(first_sum_result)):
user_sensitive=first_sum_result[i]['sensitive_num']['value']
if user_sensitive > 0:
user_dict=dict()
user_dict['uid']=first_sum_result[i]['key']
user_dict['sensitive']=user_sensitive
top_userlist.append(user_dict)
else:
pass
#查询敏感用户的敏感微博内容
results=[]
for user in top_userlist:
#print user
user_detail=dict()
user_detail['uid']=user['uid']
user_detail['user_sensitive']=user['sensitive']
user_lookup_id=xnr_uid+'_'+user['uid']
print user_lookup_id
try:
#user_result=es_xnr.get(index=weibo_feedback_follow_index_name,doc_type=weibo_feedback_follow_index_type,id=user_lookup_id)['_source']
user_result=es_user_profile.get(index=profile_index_name,doc_type=profile_index_type,id=user['uid'])['_source']
user_detail['user_name']=user_result['nick_name']
except:
user_detail['user_name']=''
query_body={
'query':{
'filtered':{
'filter':{
'bool':{
'must':[
{'term':{'uid':user['uid']}},
{'range':{'sensitive':{'gte':1}}}
]
}
}
}
},
'size':MAX_WARMING_SIZE,
'sort':{'sensitive':{'order':'desc'}}
}
try:
second_result=es_flow_text.search(index=flow_text_index_name,doc_type=flow_text_index_type,body=query_body)['hits']['hits']
except:
second_result=[]
s_result=[]
for item in second_result:
item['_source']['nick_name']=get_user_nickname(item['_source']['uid'])
s_result.append(item['_source'])
s_result.sort(key=lambda k:(k.get('sensitive',0)),reverse=True)
user_detail['content']=json.dumps(s_result)
user_detail['xnr_user_no']=xnr_user_no
user_detail['validity']=0
user_detail['timestamp']=end_time
user_detail['_id']=xnr_user_no+'_'+user_detail['uid']
results.append(user_detail)
results.sort(key=lambda k:(k.get('user_sensitive',0)),reverse=True)
return results
def weibo_user_detail(user_id):
result = es_user_profile.get(index=profile_index_name,
doc_type=profile_index_type,
id=user_id)['_source']
return result
def show_personnal_warming(xnr_user_no, day_time):
#查询关注列表
try:
es_xnr_result = es_xnr.get(
index=weibo_xnr_fans_followers_index_name,
doc_type=weibo_xnr_fans_followers_index_type,
id=xnr_user_no)['_source']
followers_list = es_xnr_result['followers_list']
except:
followers_list = []
#计算敏感度排名靠前的用户
query_body = {
'query': {
'filtered': {
'filter': {
'terms': {
'uid': followers_list
}
}
}
},
'aggs': {
'followers_sensitive_num': {
'terms': {
'field': 'uid'
},
'aggs': {
'sensitive_num': {
'sum': {
'field': 'sensitive'
}
}
}
}
},
'size': MAX_VALUE
}
#测试状态下时间设置
if S_TYPE == 'test':
test_day_date = S_DATE_BCI
test_day_time = datetime2ts(test_day_date)
flow_text_index_list = get_flow_text_index_list(test_day_time)
else:
flow_text_index_list = get_flow_text_index_list(day_time)
#print flow_text_index_list
try:
first_sum_result=es_flow_text.search(index=flow_text_index_list,doc_type=flow_text_index_type,\
body=query_body)['aggregations']['followers_sensitive_num']['buckets']
except:
first_sum_result = []
#print first_sum_result
top_userlist = []
if USER_NUM < len(first_sum_result):
temp_num = USER_NUM
else:
temp_num = len(first_sum_result)
#print temp_num
for i in xrange(0, temp_num):
user_sensitive = first_sum_result[i]['sensitive_num']['value']
if user_sensitive > 0:
user_dict = dict()
user_dict['uid'] = first_sum_result[i]['key']
user_dict['sensitive'] = user_sensitive
top_userlist.append(user_dict)
else:
pass
#查询敏感用户的最敏感微博内容
results = []
for user in top_userlist:
#print user
user_detail = dict()
user_detail['uid'] = user['uid']
user_detail['user_sensitive'] = user['sensitive']
try:
user_result = es_user_profile.get(index=profile_index_name,
doc_type=profile_index_type,
id=user['uid'])['_source']
user_detail['user_name'] = user_result['nick_name']
except:
user_detail['user_name'] = ''
query_body = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'term': {
'uid': user['uid']
}
}, {
'range': {
'sensitive': {
'gte': 1,
'lte': 100
}
}
}]
}
}
}
},
'size': USER_CONTENT_NUM,
'sort': {
'sensitive': {
'order': 'desc'
}
}
}
#if S_TYPE == 'test':
try:
second_result = es_flow_text.search(
index=flow_text_index_list,
doc_type=flow_text_index_type,
body=query_body)['hits']['hits']
except:
second_result = []
#else:
# second_result=es_xnr.search(index=weibo_xnr_flow_text_listname,doc_type=xnr_flow_text_index_type,body=query_body)['hits']['hits']
s_result = []
tem_word_one = '静坐'
tem_word_two = '集合'
for item in second_result:
sensitive_words = item['_source']['sensitive_words_string']
if ((sensitive_words == tem_word_one)
or (sensitive_words == tem_word_two)):
pass
else:
s_result.append(item['_source'])
s_result.sort(key=lambda k: (k.get('sensitive', 0)), reverse=True)
user_detail['content'] = s_result
results.append(user_detail)
results.sort(key=lambda k: (k.get('user_sensitive', 0)), reverse=True)
return results
