def lookup_date_info(account_name,start_time,end_time,today_datetime):
start_year=ts2yeartime(start_time)
end_year=ts2yeartime(end_time)
query_body={
'query':{
'match_all':{}
},
'size':MAX_VALUE,
'sort':{'date_time':{'order':'asc'}}
}
result=es_xnr.search(index=weibo_date_remind_index_name,doc_type=weibo_date_remind_index_type,body=query_body)['hits']['hits']
date_result=[]
for item in result:
date_time = item['_source']['date_time']
date_name = item['_source']['date_name']
start_tempdate_name = start_year + '-' + date_time
start_tempdate_time = datetime2ts(start_tempdate_name)
end_tempdate_name = end_year + '-' + date_time
end_tempdate_time = datetime2ts(end_tempdate_name)
year=ts2yeartime(today_datetime)
warming_date=year+'-'+date_time
item['_source']['countdown_num']=(datetime2ts(warming_date)-today_datetime)/DAY
keywords=item['_source']['keywords']
item['_source']['_id']=date_time+'_'+str(today_datetime)
# print date_time
#print start_tempdate_time,start_time,end_time,end_tempdate_time
if (start_tempdate_time >= start_time and start_tempdate_time <= end_time) or (end_tempdate_time >= start_time and end_tempdate_time <= end_time):
#print 'aaaa'
if item['_source']['create_type'] == 'all_xnrs':
item['_source']['weibo_date_warming_content']=lookup_weibo_date_warming_content(start_year,end_year,date_time,date_name,start_time,end_time,keywords)
date_result.append(item['_source'])
elif item['_source']['create_type'] == 'my_xnrs':
if item['_source']['submitter'] == account_name:
item['_source']['weibo_date_warming_content']=lookup_weibo_date_warming_content(start_year,end_year,date_time,date_name,start_time,end_time,keywords)
date_result.append(item['_source'])
else:
pass
else:
pass
date_result.sort(key=lambda k:(k.get('countdown_num',0)),reverse=True)
return date_result
def show_date_warming(today_time):
query_body = {
'query': {
'match_all': {}
},
'size': MAX_VALUE,
'sort': {
'date_time': {
'order': 'asc'
}
}
}
result = es_xnr.search(index=weibo_date_remind_index_name,
doc_type=weibo_date_remind_index_type,
body=query_body)['hits']['hits']
#取出预警时间进行处理
if S_TYPE == 'test':
today_time = 1480176000
else:
pass
# print '^^^^', len(result)
#print today_time
date_warming_result = []
for item in result:
#计算距离日期
date_time = item['_source']['date_time']
year = ts2yeartime(today_time)
warming_date = year + '-' + date_time
today_date = ts2datetime(today_time)
countdown_num = (datetime2ts(warming_date) -
datetime2ts(today_date)) / DAY
'''
if countdown_num<0:
new_warming_year=str(int(year)+1)
new_warming_date=new_warming_year+'-'+date_time
countdown_numday=(datetime2ts(new_warming_date)-datetime2ts(today_date))/DAY
item['_source']['countdown_days']=countdown_numday
else:
'''
item['_source']['countdown_days'] = countdown_num
if abs(countdown_num) < WARMING_DAY:
#根据给定的关键词查询预警微博
keywords = item['_source']['keywords']
item['_source'][
'weibo_date_warming_content'] = lookup_weibo_date_warming(
keywords, today_time)
date_warming_result.append(item['_source'])
else:
pass
return date_warming_result
def report_warming_content(task_detail):
report_dict=dict()
report_dict['report_type']=task_detail['report_type']
report_dict['report_time']=task_detail['report_time']
report_dict['xnr_user_no']=task_detail['xnr_user_no']
report_dict['event_name']=task_detail['event_name']
report_dict['uid']=task_detail['uid']
report_dict['nick_name']=get_user_nickname(task_detail['uid'])
weibo_list=[]
user_list=[]
# print 'type:',type(task_detail['weibo_info']),task_detail['weibo_info']
weibo_info=task_detail['weibo_info']
for item in weibo_info:
item['timestamp'] = int(item['timestamp'])
flow_text_index_name = flow_text_index_name_pre + ts2datetime(item['timestamp'])
try:
weibo_result=es_flow_text.get(index=flow_text_index_name,doc_type=flow_text_index_type,id=item['mid'])['_source']
weibo_result['nick_name']=get_user_nickname(weibo_result['uid'])
weibo_list.append(weibo_result)
except:
if task_detail['report_type']==u'人物':
weibo_user_warning_index_name = weibo_user_warning_index_name_pre + ts2datetime(item['timestamp'])
weibo_user_warming_id=task_detail['xnr_user_no']+'_'+task_detail['uid']
try:
weibo_user_result=es_xnr.get(index=weibo_user_warning_index_name,doc_type=weibo_user_warning_index_type,id=weibo_user_warming_id)['_source']
user_warning_content=json.dumps(weibo_user_result['content'])
for content in user_warning_content:
if content['mid'] == item['mid']:
weibo_list.append(content)
else:
pass
except:
print 'user_error!'
elif task_detail['report_type']==u'言论':
weibo_speech_warning_index_name = weibo_speech_warning_index_name_pre + ts2datetime(item['timestamp'])
try:
weibo_speech_result=es_xnr.get(index=weibo_speech_warning_index_name,doc_type=weibo_speech_warning_index_type,id=task_detail['xnr_user_no']+'_'+item['mid'])['_source']
report_dict['uid']=weibo_speech_result['uid']
weibo_list.append(weibo_speech_result)
except:
# weibo_timing_warning_index_name = weibo_timing_warning_index_name_pre + ts2datetime(item['timestamp'])
print 'speech_error!'
elif task_detail['report_type']==u'事件':
weibo_event_warning_index_name = weibo_event_warning_index_name_pre + ts2datetime(item['timestamp'])
event_warning_id = task_detail['xnr_user_no']+'_'+task_detail['event_name']
try:
event_result=es_xnr.get(index=weibo_event_warning_index_name,doc_type=weibo_event_warning_index_type,id=event_warning_id)['_source']
event_content=json.dumps(event_result['main_weibo_info'])
for event in event_content:
if event['mid'] == item['mid']:
weibo_list.append(event)
else:
pass
except:
print 'event_error!'
elif task_detail['report_type']==u'时间':
year = ts2yeartime(item['timestamp'])
weibo_timing_warning_index_name = weibo_timing_warning_index_name_pre + year +'_' + task_detail['date_time']
try:
time_result=es_xnr.search(index=weibo_timing_warning_index_name,doc_type=weibo_timing_warning_index_type,query_body={'query':{'match_all':{}}})['hits']['hits']
time_content=[]
for timedata in time_result:
for data in timedata['weibo_date_warming_content']:
if data['mid'] == item['mid']:
weibo_list.append(data)
else:
pass
except:
print 'time_error!'
user_info=task_detail['user_info']
if user_info:
for uid in user_info:
user=dict()
try:
user_result=es_user_profile.get(index=profile_index_name,doc_type=profile_index_type,id=uid)['_source']
user['uid']=uid
user['nick_name']=user_result['nick_name']
user['fansnum']=user_result['fansnum']
user['friendsnum']=user_result['friendsnum']
user['photo_url']=user_result['photo_url']
user_list.append(user)
except:
user['uid']=uid
user['nick_name']=''
user['fansnum']=0
user['friendsnum']=0
user['photo_url']=''
user_list.append(user)
print 'user_list error!'
else:
pass
report_content=dict()
report_content['user_list']=user_list
report_content['weibo_list']=weibo_list
report_dict['report_content']=json.dumps(report_content)
report_id=''
if task_detail['report_type'] == u'言论':
report_id=weibo_info[0]['mid']
elif task_detail['report_type'] == u'人物':
report_id=task_detail['xnr_user_no']+'_'+task_detail['uid']
elif task_detail['report_type'] == u'事件':
report_id=task_detail['xnr_user_no']+'_'+task_detail['event_name']
elif task_detail['report_type'] == u'时间':
# print weibo_info
if weibo_info:
report_id=weibo_info[0]['mid']
else:
report_id=str(task_detail['report_time'])
if weibo_list:
report_mark=True
else:
report_mark=False
#预警上报后不再显示问题
now_time=int(time.time())
weibo_report_management_index_name = weibo_report_management_index_name_pre + ts2datetime(now_time)
if es_xnr.indices.exists(index=weibo_report_management_index_name):
pass
else:
weibo_report_management_mappings()
if report_id and report_mark:
try:
es_xnr.index(index=weibo_report_management_index_name,doc_type=weibo_report_management_index_type,id=report_id,body=report_dict)
mark=True
except:
mark=False
else:
mark=False
return mark
def report_warming_content(task_detail):
report_dict=dict()
report_dict['report_type']=task_detail['report_type']
report_dict['report_time']=task_detail['report_time']
report_dict['xnr_user_no']=task_detail['xnr_user_no']
report_dict['event_name']=task_detail['event_name']
report_dict['uid']=task_detail['uid']
report_dict['nick_name']=get_user_nickname(task_detail['uid'])
tw_list=[]
user_list=[]
# print 'type:',type(task_detail['weibo_info']),task_detail['weibo_info']
tw_info=task_detail['tw_info']
for item in tw_info:
lookup_mark=False
item['timestamp'] = int(item['timestamp'])
if task_detail['report_type']==u'人物':
twitter_user_warning_index_name = twitter_user_warning_index_name_pre + ts2datetime(item['timestamp'])
twitter_user_warming_id=task_detail['xnr_user_no']+'_'+task_detail['uid']
try:
twitter_user_result=es_xnr_2.get(index=twitter_user_warning_index_name,doc_type=twitter_user_warning_index_type,id=twitter_user_warming_id)['_source']
user_warning_content=json.dumps(twitter_user_result['content'])
for content in user_warning_content:
if content['tid'] == item['tid']:
lookup_mark=True
tw_list.append(content)
else:
pass
except:
print 'user_error!'
elif task_detail['report_type']==u'言论':
twitter_speech_warning_index_name = twitter_speech_warning_index_name_pre + ts2datetime(item['timestamp'])
try:
twitter_speech_result=es_xnr_2.get(index=twitter_speech_warning_index_name,doc_type=twitter_speech_warning_index_type,id=task_detail['xnr_user_no']+'_'+item['tid'])['_source']
report_dict['uid']=twitter_speech_result['uid']
lookup_mark=True
tw_list.append(twitter_speech_result)
except:
# weibo_timing_warning_index_name = weibo_timing_warning_index_name_pre + ts2datetime(item['timestamp'])
print 'speech_error!'
elif task_detail['report_type']==u'事件':
twitter_event_warning_index_name = twitter_event_warning_index_name_pre + ts2datetime(item['timestamp'])
event_warning_id = task_detail['xnr_user_no']+'_'+task_detail['event_name']
try:
event_result=es_xnr_2.get(index=twitter_event_warning_index_name,doc_type=twitter_event_warning_index_type,id=event_warning_id)['_source']
event_content=json.dumps(event_result['main_twitter_info'])
for event in event_content:
if event['tid'] == item['tid']:
lookup_mark=True
tw_list.append(event)
else:
pass
except:
print 'event_error!'
elif task_detail['report_type']==u'时间':
year = ts2yeartime(item['timestamp'])
twitter_timing_warning_index_name = twitter_timing_warning_index_name_pre + year +'_' + task_detail['date_time']
try:
time_result=es_xnr_2.search(index=twitter_timing_warning_index_name,doc_type=twitter_timing_warning_index_type,query_body={'query':{'match_all':{}}})['hits']['hits']
time_content=[]
for timedata in time_result:
for data in timedata['twitter_date_warming_content']:
if data['tid'] == item['tid']:
lookup_mark=True
tw_list.append(data)
else:
pass
except:
print 'time_error!'
if lookup_mark:
pass
else:
flow_text_index_name = twitter_flow_text_index_name_pre + ts2datetime(item['timestamp'])
try:
tw_result=es_xnr_2.get(index=flow_text_index_name,doc_type=twitter_flow_text_index_type,id=item['tid'])['_source']
tw_result['nick_name']=get_user_nickname(fb_result['uid'])
tid_result=lookup_tid_attend_index(item['tid'],item['timestamp'])
if tid_result:
tw_result['comment']=tid_result['comment']
tw_result['share']=tid_result['share']
tw_result['favorite']=tid_result['favorite']
else:
tw_result['comment']=0
tw_result['share']=0
tw_result['favorite']=0
tw_list.append(tw_result)
except:
print 'flow_text error!'
user_info=task_detail['user_info']
if user_info:
for uid in user_info:
user=dict()
try:
user_result=es_xnr_2.get(index=twitter_user_index_name,doc_type=twitter_user_index_type,id=uid)['_source']
user_dict['uid']=item['_id']
user_dict['username']=user_result['username']
if user_result.has_key('talking_about_count'):
user_dict['talking_about_count']=user_result['talking_about_count']
else:
user_dict['talking_about_count']=0
if user_result.has_key('likes'):
user_dict['likes']=user_result['likes']
else:
user_dict['likes']=0
if user_result.has_key('category'):
user_dict['category']=user_result['category']
else:
user_dict['category']=''
user_list.append(user)
except:
user_dict['uid']=item['_id']
user_dict['username']=''
user_dict['talking_about_count']=0
user_dict['likes']=0
user_dict['category']=''
user_list.append(user)
print 'user_list error!'
else:
pass
report_content=dict()
report_content['user_list']=user_list
report_content['tw_list']=tw_list
report_dict['report_content']=json.dumps(report_content)
report_id=''
if task_detail['report_type'] == u'言论':
report_id=weibo_info[0]['tid']
elif task_detail['report_type'] == u'人物':
report_id=task_detail['xnr_user_no']+'_'+task_detail['uid']
elif task_detail['report_type'] == u'事件':
report_id=task_detail['xnr_user_no']+'_'+task_detail['event_name']
elif task_detail['report_type'] == u'时间':
# print weibo_info
if tw_info:
report_id=tw_info[0]['tid']
else:
report_id=str(task_detail['report_time'])
if tw_list:
report_mark=True
else:
report_mark=False
#预警上报后不再显示问题
now_time=int(time.time())
twitter_report_management_index_name = twitter_report_management_index_name_pre + ts2datetime(now_time)
if es_xnr_2.indices.exists(index=twitter_report_management_index_name):
pass
else:
twitter_report_management_mappings()
if report_id and report_mark:
try:
es_xnr_2.index(index=twitter_report_management_index_name,doc_type=twitter_report_management_index_type,id=report_id,body=report_dict)
mark=True
except:
mark=False
else:
mark=False
return mark