def utils_show_sensitive_users(wxbot_id, period, startdate='', enddate=''): start_ts, end_ts, period = dump_date(period, startdate, enddate) index_names = get_wx_groupmessage_index_list(ts2datetime(start_ts), ts2datetime(end_ts)) index_names.reverse() xnr_puid = load_wxxnr_redis_data(wxbot_id=wxbot_id, items=['puid'])['puid'] query_body = { "query": { "filtered":{ "filter":{ "bool":{ "must":[ {"term":{"xnr_id": xnr_puid}}, {"term":{"sensitive_flag": 1}}, {'term':{'msg_type':'Text'.lower()}} ] } } } }, "aggs":{ "sen_users":{ "terms":{"field": "speaker_id"} } }, "sort":{"timestamp":{"order":"desc"}} } sensitive_users = {} for index_name in index_names: try: search_result = es_xnr.search(index=index_name, doc_type=wx_group_message_index_type,body=query_body) if search_result: res = search_result['aggregations']['sen_users']['buckets'] docs = search_result['hits']['hits'] for r in res: groups_list = [] speaker_id = r['key'] count = r['doc_count'] for doc in docs: if doc['_source']['speaker_id'] == speaker_id: groups_list.append(doc['_source']['group_name']) if speaker_id in sensitive_users: #update: groups&count。因为是倒序查询,所以last_speak_ts在最初创建的时候就是最终的值,不需要更新。 sensitive_users[speaker_id]['count'] += count sensitive_users[speaker_id]['groups_list'].extend(groups_list) else: #匹配第一条即可 for doc in docs: if doc['_source']['speaker_id'] == speaker_id: nickname = doc['_source']['speaker_name'] last_speak_ts = doc['_source']['timestamp'] break sensitive_users[speaker_id] = { 'nickname': nickname, 'count': count, 'last_speak_ts': last_speak_ts, 'groups_list': groups_list } except Exception,e: pass
def get_modify_userinfo(task_detail): item_dict = {} nick_name = task_detail['nick_name'] location_list = task_detail['location'].encode('utf-8').split(',') try: item_dict['location_province'] = location_list[0] item_dict['location_city'] = location_list[1] except: item_dict['location_province'] = location_list[0] item_dict['location_city'] = location_list[0] item_dict['description'] = task_detail['description'] gender = task_detail['gender'] if gender == u'男': item_dict['gender'] = 'man' else: item_dict['gender'] = 'woman' age = task_detail['age'] birth_year = time.localtime().tm_year - int(age) month = '%02d' % random.randint(0, 13) day = '%02d' % random.randint(0, 29) item_dict['birth'] = [str(birth_year), month, day] query_body = { 'query': { 'filtered': { 'filter': { 'term': { 'nick_name': nick_name } } } } } es_results = es.search(index=fb_xnr_index_name, doc_type=fb_xnr_index_type, body=query_body)['hits']['hits'] xnr_result = es_results[0]['_source'] try: fb_mail_account = xnr_result['fb_mail_account'] except: fb_mail_account = '' try: fb_phone_account = xnr_result['fb_phone_account'] except: fb_phone_account = '' if fb_mail_account: account_name = fb_mail_account else: account_name = fb_phone_account password = xnr_result['password'] uid = xnr_result['uid'] try: result = change_userinfo(account_name, password, uid, item_dict) except: result = False return result
def get_show_example_model(): es_results = es.search(index=fb_example_model_index_name,doc_type=fb_example_model_index_type,\ body={'query':{'match_all':{}}})['hits']['hits'] result_all = [] for result in es_results: result = result['_source'] result_all.append(result) return result_all
def utils_show_report_content(wxbot_id, report_type, period, startdate, enddate): start_ts, end_ts, period = dump_date(period, startdate, enddate) xnr_puid = load_wxxnr_redis_data(wxbot_id=wxbot_id, items=['puid'])['puid'] result = [] query_body = { 'query': { 'bool': { 'must': [{ 'term': { 'report_type': report_type } }, { 'range': { 'report_time': { 'gte': start_ts, 'lte': end_ts } } }, { 'term': { 'xnr_user_no': wxbot_id } }, { 'term': { 'xnr_puid': xnr_puid } }] } }, 'size': MAX_SEARCH_SIZE, 'sort': [{ 'report_time': { 'order': 'desc' } }] } try: wx_report_management_mappings() es_result = es_xnr.search(index=wx_report_management_index_name, doc_type=wx_report_management_index_type, body=query_body)['hits']['hits'] if es_result: for item in es_result: try: data = item['_source'] data['_id'] = item['_id'] report_content = eval(item['_source']['report_content']) data['sensitive_value'] = report_content['sensitive_value'] data['text'] = report_content['text'] data['sensitive_words_string'] = report_content[ 'sensitive_words_string'].decode('utf8') data.pop('report_content') result.append(data) except: pass except Exception, e: print 'wx_report_management Exception: ', str(e)
def get_community_warning(xnr_user_no, community_id, start_time, end_time): if S_TYPE == 'test': time_gap = end_time - start_time end_time = datetime2ts(WEIBO_COMMUNITY_DATE) start_time = end_time - time_gap else: pass weibo_trace_community_index_name = weibo_trace_community_index_name_pre + xnr_user_no.lower( ) # print 'weibo_trace_community_index_name:',weibo_trace_community_index_name print 'time:', ts2datetime(start_time), ts2datetime(end_time) query_body = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'term': { 'community_id': community_id } }, { 'range': { 'trace_time': { 'gte': start_time, 'lte': end_time } } }] } } } }, 'size': 30, 'sort': { 'trace_time': { 'order': 'asc' } } } try: community_result = es_xnr.search(index=weibo_trace_community_index_name,\ doc_type=weibo_trace_community_index_type,body=query_body)['hits']['hits'] result = [] # print 'community_result::',community_result for item in community_result: result.append(item['_source']) except: result = [] if result: #对预警内容进行融合 result.sort(key=lambda k: (k.get('trace_time', 0)), reverse=False) warning_result = get_warning_orgnize(result) else: warning_result = get_newcommunity_warning(xnr_user_no, community_id, start_time, end_time) return warning_result
def show_authority_list(): query_body = {'query': {'match_all': {}}, 'size': MAX_VALUE} result = es.search(index=weibo_authority_management_index_name, doc_type=weibo_authority_management_index_type, body=query_body)['hits']['hits'] results = [] for item in result: results.append(item['_source']) return results
def get_show_domain(): domain_name_dict = {} query_body = {'query':{'match_all':{}},'size':MAX_SEARCH_SIZE} es_results = es.search(index=tw_domain_index_name,doc_type=tw_domain_index_type,body=query_body)['hits']['hits'] if es_results: for result in es_results: result = result['_source'] domain_name_dict[result['domain_pinyin']] = result['domain_name'] return domain_name_dict
def get_show_domain(): domain_name_dict = {} query_body = {'query':{'term':{'compute_status':3}},'size':MAX_SEARCH_SIZE} es_results = es.search(index=weibo_domain_index_name,doc_type=weibo_domain_index_type,body=query_body)['hits']['hits'] if es_results: for result in es_results: result = result['_source'] domain_name_dict[result['domain_pinyin']] = result['domain_name'] return domain_name_dict
def utils_get_safe(wxbot_id, period, startdate, enddate): start_ts, end_ts, period = dump_date(period, startdate, enddate) current_timestamp = int(time.time()) current_date = ts2datetime(current_timestamp) if period == 0: #获取今天的数据 current_time = datetime2ts(current_date) last_date = ts2datetime(current_time-DAY) speak_dict = {} speak_dict['speak_day'] = {} speak_dict['speak_total'] = {} xnr_puid = load_wxxnr_redis_data(wxbot_id=wxbot_id, items=['puid'])['puid'] #获取xnr今日发言总数 today_count = 0 query_body = { 'query':{ 'bool':{ 'must':[ {'term':{'speaker_id': xnr_puid}}, {'term':{'xnr_id':xnr_puid}} ] } } } today_index_name = wx_group_message_index_name_pre + current_date try: today_count_result = es_xnr.count(index=today_index_name,doc_type=wx_group_message_index_type,body=query_body) if today_count_result['_shards']['successful'] != 0: today_count = today_count_result['count'] except Exception,e: pass #print 'today_count Exception: ', str(e) #获取xnr历史发言总数 total_count = 0 total_query_body = { 'query':{ 'bool':{ 'must':[ {'term':{'xnr_user_no': wxbot_id}}, {'term':{'puid':xnr_puid}}, {'term':{'date_time':last_date}} ] } } } total_index_name = wx_xnr_history_count_index_name try: total_count_result = es_xnr.search(index=total_index_name,doc_type=wx_xnr_history_count_index_type,body=total_query_body) if total_count_result['_shards']['successful'] != 0: total_count = total_count_result['hits']['hits'][0]['_source']['total_post_num'] except Exception,e: pass
def lookup_hot_posts(start_time,end_time,weiboxnr_id,classify_id,order_id): #weiboxnr_id,classify_id暂时不用 print "weiboxnr_id------------------------------------------------------------" print weiboxnr_id print 'post request start!!!!!' weibo_sensitive_post_index_name_list = [] days_num = int((end_time - start_time)/DAY) + 1 print 'days_num = int((end_time - start_time)/DAY) + 1-=-=-=' print end_time, start_time, DAY print days_num for i in range(0,days_num): date_ts = end_time - i*DAY print 'date_ts, end_time, i, DAY-=-=-=-=-=-=-=-=-=-=-' print date_ts, end_time, i, DAY index_name = weibo_sensitive_post_index_name_pre + ts2datetime(date_ts) print index_name if es_xnr.indices.exists(index=index_name): weibo_sensitive_post_index_name_list.append(index_name) print 'has append -----------0000000000000000000000000000' print 'has not append -----------0000000000000000000000000000' sort_condition_list = [] if order_id==1: #按时间排序 sort_condition_list=[{'timestamp':{'order':'desc'}}] elif order_id==2: #按热度排序 sort_condition_list=[{'retweeted':{'order':'desc'}}] elif order_id==3: #按敏感度排序 sort_condition_list=[{'sensitive':{'order':'desc'}}] query_body={ 'query':{ 'filtered':{ 'filter':{ 'bool':{ 'must':[{'range':{'timestamp':{'gte':int(start_time),'lt':int(end_time)}}}] } } } }, 'size':HOT_WEIBO_NUM, 'sort':sort_condition_list } try: es_result=es_xnr.search(index=weibo_sensitive_post_index_name_list,doc_type=weibo_sensitive_post_index_type,\ body=query_body)['hits']['hits'] hot_result=[] for item in es_result: hot_result.append(item['_source']) except: hot_result=[] return hot_result
def utils_search_by_group_puid(wxbot_id, group_puid, period, startdate='', enddate=''): start_ts, end_ts, period = dump_date(period, startdate, enddate) index_names = get_wx_groupmessage_index_list(ts2datetime(start_ts), ts2datetime(end_ts)) index_names.reverse() xnr_puid = load_wxxnr_redis_data(wxbot_id=wxbot_id, items=['puid'])['puid'] query_body = { "query": { "filtered": { "filter": { "bool": { "must": [ { "term": { "xnr_id": xnr_puid } }, { 'term': { 'group_id': group_puid } }, { "terms": { "msg_type": [ 'Text'.lower(), 'Picture'.lower(), 'Recording'.lower() ] } }, ] } } } }, "size": MAX_VALUE, "sort": { "timestamp": { "order": "desc" } } } results = [] for index_name in index_names: try: search_result = es_xnr.search(index=index_name, doc_type=wx_group_message_index_type, body=query_body) if search_result: results.extend(search_result['hits']['hits']) except Exception, e: pass
def get_tw_xnr_no(): user_no_max = 0 if not r.exists(tw_xnr_max_no): #如果当前redis没有记录,则去es数据库查找补上 es_results = es.search(index=tw_xnr_index_name,doc_type=tw_xnr_index_type,body={'query':{'match_all':{}},\ 'sort':{'user_no':{'order':'desc'}}})['hits']['hits'] if es_results: user_no_max = es_results[0]['_source']['user_no'] else: #如果当前redis有记录,则取用并更新 user_no_max = int(r.get(tw_xnr_max_no)) return user_no_max
def search_by_xnr_number(xnr_qq_number, current_date,group_qq_name): group_qq_name_list = group_qq_name.encode('utf-8').split(',') # 用于显示操作页面初始的所有群历史信息 query_body = { "query": { "filtered":{ "filter":{ "bool":{ "must":[ {"term":{"xnr_qq_number":xnr_qq_number}}, {'terms':{'qq_group_nickname':group_qq_name_list}} ] } } } }, "size": MAX_VALUE, "sort":{"timestamp":{"order":"desc"}} } enddate = current_date startdate = ts2datetime(datetime2ts(enddate)-group_message_windowsize*DAY) index_names = get_groupmessage_index_list(startdate,enddate) print 'index_names::',index_names index_names.reverse() results = {} for index_name in index_names: # if not es_xnr.indices.exsits(index=index_name): # continue try: result = es_xnr.search(index=index_name, doc_type=group_message_index_type,body=query_body) if results != {}: results['hits']['hits'].extend(result['hits']['hits']) else: results=result #.copy() except: pass # results_new = [] # for index_name in index_names: # try: # es_results = es_xnr.search(index=index_name, doc_type=group_message_index_type,body=query_body)['hits']['hits'] # print 'es_results::',es_results # for es_result in es_results: # es_result = es_result['_source'] # results_new.append(es_result) # except: # continue return results
def show_reportcontent_new(report_type, start_time, end_time): query_condition = [] # report_type_content=json.loads(report_type) # print 'report_type_content:',report_type_content if report_type: query_condition.append({'terms': {'report_type': report_type}}) else: pass query_condition.append( {'range': { 'report_time': { 'gte': start_time, 'lte': end_time } }}) query_body = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': query_condition } } } }, 'size': MAX_SEARCH_SIZE, 'sort': { 'report_time': { 'order': 'desc' } } } report_management_index_name = get_xnr_reportment_index_listname( weibo_report_management_index_name_pre, start_time, end_time) result = [] #print 'time::',start_time,end_time #print 'index::',report_management_index_name try: if report_management_index_name: results = es_xnr.search( index=report_management_index_name, doc_type=weibo_report_management_index_type, body=query_body)['hits']['hits'] else: results = [] for item in results: item['_source']['_id'] = item['_id'] item['_source']['report_content'] = json.loads( item['_source']['report_content']) result.append(item['_source']) except: result = [] return result
def get_speaker_info(qq_nick, index_name): print 'qq_nick:', qq_nick query_body = { "query": { "filtered": { "filter": { "bool": { "must": [{ "term": { "speaker_nickname": qq_nick } }, { "term": { "sensitive_flag": 1 } }] } } } }, "size": MAX_VALUE, "sort": { "timestamp": { "order": "desc" } } } result = es_xnr.search(index=index_name, doc_type=group_message_index_type, body=query_body)['hits']['hits'] results = {} # print 'result:',result source = result[0]['_source'] if source != []: results['qq_nick'] = source['speaker_nickname'] results['qq_number'] = source['speaker_qq_number'] results['last_speak_ts'] = source['timestamp'] results['qq_groups'] = { source['qq_group_number']: source['qq_group_nickname'] } for item in result: source = item['_source'] text_item = [ source['text'], source['timestamp'], source['sensitive_words_string'] ] #print 'text_item:', text_item try: results['text'].append(text_item) except: results['text'] = [text_item] #print 'final results:', results return results
def search_by_xnr_number(xnr_qq_number, current_date): # 用于显示操作页面初始的所有群历史信息 query_body = { "query": { "filtered": { "filter": { "bool": { "must": [{ "term": { "xnr_qq_number": xnr_qq_number }, "term": { "sensitive_flag": 1 } }] } } } }, "size": MAX_VALUE, "sort": { "sensitive_value": { "order": "desc" } } } enddate = current_date startdate = ts2datetime( datetime2ts(enddate) - group_message_windowsize * DAY) index_names = get_groupmessage_index_list(startdate, enddate) # print index_names results = [] for index_name in index_names: # if not es_xnr.indices.exsits(index=index_name): # continue try: result = es_xnr.search(index=index_name, doc_type=group_message_index_type, body=query_body)['hits']['hits'] # if results != {}: # results['hits']['hits'].extend(result['hits']['hits']) # else: # results=result.copy() if result: for item in result: item['_source']['_id'] = item['_id'] results.append(item['_source']) else: pass except: pass # print 'results:',results return results
def search_by_period(xnr_qq_number, startdate, enddate, group_qq_name): group_qq_name_list = group_qq_name.encode('utf-8').split(',') results = {} query_body = { "query": { "filtered": { "filter": { "bool": { "must": [{ "term": { "xnr_qq_number": xnr_qq_number } }, { 'terms': { 'qq_group_nickname': group_qq_name_list } }] } } } }, "size": MAX_VALUE, "sort": { "timestamp": { "order": "desc" } } } # es.search(index=”flow_text_2013-09-02”, doc_type=”text”, body=query_body) index_names = get_groupmessage_index_list(startdate, enddate) index_names.reverse() #print 'index_names::',index_names for index_name in index_names: # if not es_xnr.indices.exsits(index_name): # continue print index_name try: result = es_xnr.search(index=index_name, doc_type=group_message_index_type, body=query_body) print result if results != {}: results['hits']['hits'].extend(result['hits']['hits']) else: results = result.copy() except: pass if results == {}: results = {'hits': {'hits': []}} return results
def show_date_warming(today_time): query_body = { 'query': { 'match_all': {} }, 'size': MAX_VALUE, 'sort': { 'date_time': { 'order': 'asc' } } } result = es_xnr.search(index=weibo_date_remind_index_name, doc_type=weibo_date_remind_index_type, body=query_body)['hits']['hits'] #取出预警时间进行处理 if S_TYPE == 'test': today_time = 1480176000 else: pass # print '^^^^', len(result) #print today_time date_warming_result = [] for item in result: #计算距离日期 date_time = item['_source']['date_time'] year = ts2yeartime(today_time) warming_date = year + '-' + date_time today_date = ts2datetime(today_time) countdown_num = (datetime2ts(warming_date) - datetime2ts(today_date)) / DAY ''' if countdown_num<0: new_warming_year=str(int(year)+1) new_warming_date=new_warming_year+'-'+date_time countdown_numday=(datetime2ts(new_warming_date)-datetime2ts(today_date))/DAY item['_source']['countdown_days']=countdown_numday else: ''' item['_source']['countdown_days'] = countdown_num if abs(countdown_num) < WARMING_DAY: #根据给定的关键词查询预警微博 keywords = item['_source']['keywords'] item['_source'][ 'weibo_date_warming_content'] = lookup_weibo_date_warming( keywords, today_time) date_warming_result.append(item['_source']) else: pass return date_warming_result
def cron_compute_mark_wx(): current_time = int(time.time() - DAY) current_date = ts2datetime(current_time) current_time_new = datetime2ts(current_date) #加载数据库中虚拟人信息 xnr_query_body = {'query': {'match_all': {}}, 'size': MAX_VALUE} xnr_results = es_xnr.search(index=wx_xnr_index_name, doc_type=wx_xnr_index_type, body=xnr_query_body)['hits']['hits'] flag = False for result in xnr_results: xnr_user_no = result['_source']['xnr_user_no'] puid = result['_source']['puid'] #计算 influence_dict = get_influence_at_num(puid) penetration_dict = get_penetration_num(xnr_user_no) safe_dict = wx_history_count(xnr_user_no, puid, current_time_new) #整理 _id = xnr_user_no + '_' + current_date xnr_user_detail = { 'influence': influence_dict['mark'], 'penetration': penetration_dict['mark'], 'safe': safe_dict['mark'], 'daily_be_at_num': influence_dict['daily_be_at_num'], 'total_be_at_num': influence_dict['total_be_at_num'], 'daily_sensitive_num': penetration_dict['sensitive_info'], 'total_post_num': safe_dict['total_post_num'], 'daily_post_num': safe_dict['daily_post_num'], 'date_time': current_date, 'timestamp': current_time_new, 'xnr_user_no': xnr_user_no, 'puid': puid } #并存储 wx_xnr_history_count_mappings() #先确保数据库存在 try: if es_xnr.exists(index=wx_xnr_history_count_index_name, doc_type=wx_xnr_history_count_index_type, id=_id): msg = es_xnr.update(index=wx_xnr_history_count_index_name, doc_type=wx_xnr_history_count_index_type, body={'doc': xnr_user_detail}, id=_id) else: msg = es_xnr.index(index=wx_xnr_history_count_index_name, doc_type=wx_xnr_history_count_index_type, id=_id, body=xnr_user_detail) flag = True print msg except Exception, e: print 'cron_compute_mark_wx Exception: ', str(e) return False
def get_show_example_model(): #print '!!!!!',{'query':{'term':{'xnr_user_no':xnr_user_no}}} es_results = es.search(index=weibo_example_model_index_name,doc_type=weibo_example_model_index_type,\ body={'query':{'match_all':{}}})['hits']['hits'] result_all = [] for result in es_results: result = result['_source'] result_all.append(result) return result_all
def get_xnr_info(task_detail): nick_name = task_detail['nick_name'] query_body = { 'query':{ 'filtered':{ 'filter':{ 'term':{'nick_name':nick_name} } } } } es_results = es.search(index=tw_xnr_index_name,doc_type=tw_xnr_index_type,body=query_body)['hits']['hits'] return es_results
def get_nick_name_unique(nick_name): query_body = { 'query':{ 'term':{'nick_name':nick_name} } } es_profile_results = es_user_profile.search(index=profile_index_name,doc_type=profile_index_type,body=query_body)['hits']['hits'] es_xnr_results = es.search(index=tw_xnr_index_name,doc_type=tw_xnr_index_type,body=query_body)['hits']['hits'] if es_profile_results and es_xnr_results: mark = False else: mark = True return mark
def search_by_period(xnr_qq_number, startdate, enddate): results = [] query_body = { "query": { "filtered": { "filter": { "bool": { "must": [{ "term": { "xnr_qq_number": xnr_qq_number }, "term": { "sensitive_flag": 1 } }] } } } }, "size": MAX_VALUE, "sort": { "timestamp": { "order": "desc" } } } # es.search(index=”flow_text_2013-09-02”, doc_type=”text”, body=query_body) index_names = get_groupmessage_index_list(startdate, enddate) for index_name in index_names: # if not es_xnr.indices.exsits(index_name): # continue try: result = es_xnr.search(index=index_name, doc_type=group_message_index_type, body=query_body)['hits']['hits'] # if results != {}: # results['hits']['hits'].extend(result['hits']['hits']) # else: # results=result.copy() if result: for item in result: item['_source']['_id'] = item['_id'] results.append(item['_source']) else: pass except: pass # if results == {}: # results={'hits':{'hits':[]}} return results
def get_penetration_num(xnr_user_no): current_timestamp = int(time.time() - DAY) current_date = ts2datetime(current_timestamp) current_time = datetime2ts(current_date) xnr_data = load_wxxnr_redis_data(wxbot_id=xnr_user_no, items=['puid', 'groups_list']) puid = xnr_data['puid'] group_list = xnr_data['groups_list'] #查询1 sensitive_value = 0 wx_group_message_index_name = wx_group_message_index_name_pre + current_date query_body_info = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'terms': { 'group_id': group_list } }, { 'range': { 'sensitive_value': { 'gte': -1 } } }] } } } }, 'aggs': { 'avg_sensitive': { 'avg': { 'field': 'sensitive_value' } } } } try: es_sensitive_result = es_xnr.search( index=wx_group_message_index_name, doc_type=wx_group_message_index_type, body=query_body_info)['aggregations'] sensitive_value = es_sensitive_result['avg_sensitive']['value'] if sensitive_value == None: sensitive_value = 0 except Exception, e: print 'sensitive_value Exception: ', str(e)
def show_hidden_expression(): query_body={ 'query':{ 'match_all':{} }, 'size':MAX_VALUE, 'sort':{'create_time':{'order':'desc'}} } result=es.search(index=weibo_hidden_expression_index_name,doc_type=weibo_hidden_expression_index_type,body=query_body)['hits']['hits'] results=[] for item in result: item['_source']['id']=item['_id'] results.append(item['_source']) return results
def show_sensitive_words_default(): query_body={ 'query':{ 'match_all':{} }, 'size':MAX_SEARCH_SIZE, 'sort':{'create_time':{'order':'desc'}} } result=es.search(index=weibo_sensitive_words_index_name,doc_type=weibo_sensitive_words_index_type,body=query_body)['hits']['hits'] results=[] for item in result: item['_source']['id']=item['_id'] results.append(item['_source']) return results
def get_user_text(qq_nick, qq_groups, last_speak_ts): print 'qq_nick:', qq_nick query_body = { "query": { "filtered": { "filter": { "bool": { "must": [{ "term": { "speaker_nickname": qq_nick } }, { "term": { "sensitive_flag": 1 } }, { "term": { "qq_group_nickname": qq_groups } }] } } } }, "size": MAX_VALUE, "sort": { "timestamp": { "order": "desc" } } } index_name = group_message_index_name_pre + ts2datetime(last_speak_ts) result = es_xnr.search(index=index_name, doc_type=group_message_index_type, body=query_body)['hits']['hits'] results = {} for item in result: source = item['_source'] text_item = [ source['text'], source['timestamp'], source['sensitive_words_string'] ] #print 'text_item:', text_item try: results['text'].append(text_item) except: results['text'] = [text_item] #print 'final results:', results return results
def get_all_ports(): query_body = {"query": {"match_all": {}}, "size": MAX_VALUE} try: result = es_xnr.search(index=wx_xnr_index_name, doc_type=wx_xnr_index_type, body=query_body)['hits']['hits'] except: return [] results = [] if result != []: for item in result: if item['_source']['wxbot_port'] not in results: results.append(int(item['_source']['wxbot_port'])) return results
def search_qq_xnr(qq_number): query_body = { "query": { "filtered":{ "filter":{ "term":{"qq_number": qq_number} } } }, 'size':MAX_VALUE } result = es_xnr.search(index=qq_xnr_index_name, doc_type=qq_xnr_index_type, body=query_body) return result
def show_report_content(): query_body={ 'query':{ 'match_all':{} }, 'size':MAX_SEARCH_SIZE, 'sort':{'report_time':{'order':'desc'}} } results=es_xnr.search(index=weibo_report_management_index_name,doc_type=weibo_report_management_index_type,body=query_body)['hits']['hits'] result=[] for item in results: item['_source']['_id']=item['_id'] item['_source']['report_content']=json.loads(item['_source']['report_content']) result.append(item['_source']) return result