def lookup_date_info(account_name,start_time,end_time,today_datetime): start_year=ts2yeartime(start_time) end_year=ts2yeartime(end_time) query_body={ 'query':{ 'match_all':{} }, 'size':MAX_VALUE, 'sort':{'date_time':{'order':'asc'}} } result=es_xnr.search(index=weibo_date_remind_index_name,doc_type=weibo_date_remind_index_type,body=query_body)['hits']['hits'] date_result=[] for item in result: date_time = item['_source']['date_time'] date_name = item['_source']['date_name'] start_tempdate_name = start_year + '-' + date_time start_tempdate_time = datetime2ts(start_tempdate_name) end_tempdate_name = end_year + '-' + date_time end_tempdate_time = datetime2ts(end_tempdate_name) year=ts2yeartime(today_datetime) warming_date=year+'-'+date_time item['_source']['countdown_num']=(datetime2ts(warming_date)-today_datetime)/DAY keywords=item['_source']['keywords'] item['_source']['_id']=date_time+'_'+str(today_datetime) # print date_time #print start_tempdate_time,start_time,end_time,end_tempdate_time if (start_tempdate_time >= start_time and start_tempdate_time <= end_time) or (end_tempdate_time >= start_time and end_tempdate_time <= end_time): #print 'aaaa' if item['_source']['create_type'] == 'all_xnrs': item['_source']['weibo_date_warming_content']=lookup_weibo_date_warming_content(start_year,end_year,date_time,date_name,start_time,end_time,keywords) date_result.append(item['_source']) elif item['_source']['create_type'] == 'my_xnrs': if item['_source']['submitter'] == account_name: item['_source']['weibo_date_warming_content']=lookup_weibo_date_warming_content(start_year,end_year,date_time,date_name,start_time,end_time,keywords) date_result.append(item['_source']) else: pass else: pass date_result.sort(key=lambda k:(k.get('countdown_num',0)),reverse=True) return date_result
def show_date_warming(today_time): query_body = { 'query': { 'match_all': {} }, 'size': MAX_VALUE, 'sort': { 'date_time': { 'order': 'asc' } } } result = es_xnr.search(index=weibo_date_remind_index_name, doc_type=weibo_date_remind_index_type, body=query_body)['hits']['hits'] #取出预警时间进行处理 if S_TYPE == 'test': today_time = 1480176000 else: pass # print '^^^^', len(result) #print today_time date_warming_result = [] for item in result: #计算距离日期 date_time = item['_source']['date_time'] year = ts2yeartime(today_time) warming_date = year + '-' + date_time today_date = ts2datetime(today_time) countdown_num = (datetime2ts(warming_date) - datetime2ts(today_date)) / DAY ''' if countdown_num<0: new_warming_year=str(int(year)+1) new_warming_date=new_warming_year+'-'+date_time countdown_numday=(datetime2ts(new_warming_date)-datetime2ts(today_date))/DAY item['_source']['countdown_days']=countdown_numday else: ''' item['_source']['countdown_days'] = countdown_num if abs(countdown_num) < WARMING_DAY: #根据给定的关键词查询预警微博 keywords = item['_source']['keywords'] item['_source'][ 'weibo_date_warming_content'] = lookup_weibo_date_warming( keywords, today_time) date_warming_result.append(item['_source']) else: pass return date_warming_result
def report_warming_content(task_detail): report_dict=dict() report_dict['report_type']=task_detail['report_type'] report_dict['report_time']=task_detail['report_time'] report_dict['xnr_user_no']=task_detail['xnr_user_no'] report_dict['event_name']=task_detail['event_name'] report_dict['uid']=task_detail['uid'] report_dict['nick_name']=get_user_nickname(task_detail['uid']) weibo_list=[] user_list=[] # print 'type:',type(task_detail['weibo_info']),task_detail['weibo_info'] weibo_info=task_detail['weibo_info'] for item in weibo_info: item['timestamp'] = int(item['timestamp']) flow_text_index_name = flow_text_index_name_pre + ts2datetime(item['timestamp']) try: weibo_result=es_flow_text.get(index=flow_text_index_name,doc_type=flow_text_index_type,id=item['mid'])['_source'] weibo_result['nick_name']=get_user_nickname(weibo_result['uid']) weibo_list.append(weibo_result) except: if task_detail['report_type']==u'人物': weibo_user_warning_index_name = weibo_user_warning_index_name_pre + ts2datetime(item['timestamp']) weibo_user_warming_id=task_detail['xnr_user_no']+'_'+task_detail['uid'] try: weibo_user_result=es_xnr.get(index=weibo_user_warning_index_name,doc_type=weibo_user_warning_index_type,id=weibo_user_warming_id)['_source'] user_warning_content=json.dumps(weibo_user_result['content']) for content in user_warning_content: if content['mid'] == item['mid']: weibo_list.append(content) else: pass except: print 'user_error!' elif task_detail['report_type']==u'言论': weibo_speech_warning_index_name = weibo_speech_warning_index_name_pre + ts2datetime(item['timestamp']) try: weibo_speech_result=es_xnr.get(index=weibo_speech_warning_index_name,doc_type=weibo_speech_warning_index_type,id=task_detail['xnr_user_no']+'_'+item['mid'])['_source'] report_dict['uid']=weibo_speech_result['uid'] weibo_list.append(weibo_speech_result) except: # weibo_timing_warning_index_name = weibo_timing_warning_index_name_pre + ts2datetime(item['timestamp']) print 'speech_error!' elif task_detail['report_type']==u'事件': weibo_event_warning_index_name = weibo_event_warning_index_name_pre + ts2datetime(item['timestamp']) event_warning_id = task_detail['xnr_user_no']+'_'+task_detail['event_name'] try: event_result=es_xnr.get(index=weibo_event_warning_index_name,doc_type=weibo_event_warning_index_type,id=event_warning_id)['_source'] event_content=json.dumps(event_result['main_weibo_info']) for event in event_content: if event['mid'] == item['mid']: weibo_list.append(event) else: pass except: print 'event_error!' elif task_detail['report_type']==u'时间': year = ts2yeartime(item['timestamp']) weibo_timing_warning_index_name = weibo_timing_warning_index_name_pre + year +'_' + task_detail['date_time'] try: time_result=es_xnr.search(index=weibo_timing_warning_index_name,doc_type=weibo_timing_warning_index_type,query_body={'query':{'match_all':{}}})['hits']['hits'] time_content=[] for timedata in time_result: for data in timedata['weibo_date_warming_content']: if data['mid'] == item['mid']: weibo_list.append(data) else: pass except: print 'time_error!' user_info=task_detail['user_info'] if user_info: for uid in user_info: user=dict() try: user_result=es_user_profile.get(index=profile_index_name,doc_type=profile_index_type,id=uid)['_source'] user['uid']=uid user['nick_name']=user_result['nick_name'] user['fansnum']=user_result['fansnum'] user['friendsnum']=user_result['friendsnum'] user['photo_url']=user_result['photo_url'] user_list.append(user) except: user['uid']=uid user['nick_name']='' user['fansnum']=0 user['friendsnum']=0 user['photo_url']='' user_list.append(user) print 'user_list error!' else: pass report_content=dict() report_content['user_list']=user_list report_content['weibo_list']=weibo_list report_dict['report_content']=json.dumps(report_content) report_id='' if task_detail['report_type'] == u'言论': report_id=weibo_info[0]['mid'] elif task_detail['report_type'] == u'人物': report_id=task_detail['xnr_user_no']+'_'+task_detail['uid'] elif task_detail['report_type'] == u'事件': report_id=task_detail['xnr_user_no']+'_'+task_detail['event_name'] elif task_detail['report_type'] == u'时间': # print weibo_info if weibo_info: report_id=weibo_info[0]['mid'] else: report_id=str(task_detail['report_time']) if weibo_list: report_mark=True else: report_mark=False #预警上报后不再显示问题 now_time=int(time.time()) weibo_report_management_index_name = weibo_report_management_index_name_pre + ts2datetime(now_time) if es_xnr.indices.exists(index=weibo_report_management_index_name): pass else: weibo_report_management_mappings() if report_id and report_mark: try: es_xnr.index(index=weibo_report_management_index_name,doc_type=weibo_report_management_index_type,id=report_id,body=report_dict) mark=True except: mark=False else: mark=False return mark
def report_warming_content(task_detail): report_dict=dict() report_dict['report_type']=task_detail['report_type'] report_dict['report_time']=task_detail['report_time'] report_dict['xnr_user_no']=task_detail['xnr_user_no'] report_dict['event_name']=task_detail['event_name'] report_dict['uid']=task_detail['uid'] report_dict['nick_name']=get_user_nickname(task_detail['uid']) tw_list=[] user_list=[] # print 'type:',type(task_detail['weibo_info']),task_detail['weibo_info'] tw_info=task_detail['tw_info'] for item in tw_info: lookup_mark=False item['timestamp'] = int(item['timestamp']) if task_detail['report_type']==u'人物': twitter_user_warning_index_name = twitter_user_warning_index_name_pre + ts2datetime(item['timestamp']) twitter_user_warming_id=task_detail['xnr_user_no']+'_'+task_detail['uid'] try: twitter_user_result=es_xnr_2.get(index=twitter_user_warning_index_name,doc_type=twitter_user_warning_index_type,id=twitter_user_warming_id)['_source'] user_warning_content=json.dumps(twitter_user_result['content']) for content in user_warning_content: if content['tid'] == item['tid']: lookup_mark=True tw_list.append(content) else: pass except: print 'user_error!' elif task_detail['report_type']==u'言论': twitter_speech_warning_index_name = twitter_speech_warning_index_name_pre + ts2datetime(item['timestamp']) try: twitter_speech_result=es_xnr_2.get(index=twitter_speech_warning_index_name,doc_type=twitter_speech_warning_index_type,id=task_detail['xnr_user_no']+'_'+item['tid'])['_source'] report_dict['uid']=twitter_speech_result['uid'] lookup_mark=True tw_list.append(twitter_speech_result) except: # weibo_timing_warning_index_name = weibo_timing_warning_index_name_pre + ts2datetime(item['timestamp']) print 'speech_error!' elif task_detail['report_type']==u'事件': twitter_event_warning_index_name = twitter_event_warning_index_name_pre + ts2datetime(item['timestamp']) event_warning_id = task_detail['xnr_user_no']+'_'+task_detail['event_name'] try: event_result=es_xnr_2.get(index=twitter_event_warning_index_name,doc_type=twitter_event_warning_index_type,id=event_warning_id)['_source'] event_content=json.dumps(event_result['main_twitter_info']) for event in event_content: if event['tid'] == item['tid']: lookup_mark=True tw_list.append(event) else: pass except: print 'event_error!' elif task_detail['report_type']==u'时间': year = ts2yeartime(item['timestamp']) twitter_timing_warning_index_name = twitter_timing_warning_index_name_pre + year +'_' + task_detail['date_time'] try: time_result=es_xnr_2.search(index=twitter_timing_warning_index_name,doc_type=twitter_timing_warning_index_type,query_body={'query':{'match_all':{}}})['hits']['hits'] time_content=[] for timedata in time_result: for data in timedata['twitter_date_warming_content']: if data['tid'] == item['tid']: lookup_mark=True tw_list.append(data) else: pass except: print 'time_error!' if lookup_mark: pass else: flow_text_index_name = twitter_flow_text_index_name_pre + ts2datetime(item['timestamp']) try: tw_result=es_xnr_2.get(index=flow_text_index_name,doc_type=twitter_flow_text_index_type,id=item['tid'])['_source'] tw_result['nick_name']=get_user_nickname(fb_result['uid']) tid_result=lookup_tid_attend_index(item['tid'],item['timestamp']) if tid_result: tw_result['comment']=tid_result['comment'] tw_result['share']=tid_result['share'] tw_result['favorite']=tid_result['favorite'] else: tw_result['comment']=0 tw_result['share']=0 tw_result['favorite']=0 tw_list.append(tw_result) except: print 'flow_text error!' user_info=task_detail['user_info'] if user_info: for uid in user_info: user=dict() try: user_result=es_xnr_2.get(index=twitter_user_index_name,doc_type=twitter_user_index_type,id=uid)['_source'] user_dict['uid']=item['_id'] user_dict['username']=user_result['username'] if user_result.has_key('talking_about_count'): user_dict['talking_about_count']=user_result['talking_about_count'] else: user_dict['talking_about_count']=0 if user_result.has_key('likes'): user_dict['likes']=user_result['likes'] else: user_dict['likes']=0 if user_result.has_key('category'): user_dict['category']=user_result['category'] else: user_dict['category']='' user_list.append(user) except: user_dict['uid']=item['_id'] user_dict['username']='' user_dict['talking_about_count']=0 user_dict['likes']=0 user_dict['category']='' user_list.append(user) print 'user_list error!' else: pass report_content=dict() report_content['user_list']=user_list report_content['tw_list']=tw_list report_dict['report_content']=json.dumps(report_content) report_id='' if task_detail['report_type'] == u'言论': report_id=weibo_info[0]['tid'] elif task_detail['report_type'] == u'人物': report_id=task_detail['xnr_user_no']+'_'+task_detail['uid'] elif task_detail['report_type'] == u'事件': report_id=task_detail['xnr_user_no']+'_'+task_detail['event_name'] elif task_detail['report_type'] == u'时间': # print weibo_info if tw_info: report_id=tw_info[0]['tid'] else: report_id=str(task_detail['report_time']) if tw_list: report_mark=True else: report_mark=False #预警上报后不再显示问题 now_time=int(time.time()) twitter_report_management_index_name = twitter_report_management_index_name_pre + ts2datetime(now_time) if es_xnr_2.indices.exists(index=twitter_report_management_index_name): pass else: twitter_report_management_mappings() if report_id and report_mark: try: es_xnr_2.index(index=twitter_report_management_index_name,doc_type=twitter_report_management_index_type,id=report_id,body=report_dict) mark=True except: mark=False else: mark=False return mark