def utils_show_sensitive_users(wxbot_id, period, startdate='', enddate=''):
start_ts, end_ts, period = dump_date(period, startdate, enddate)
index_names = get_wx_groupmessage_index_list(ts2datetime(start_ts), ts2datetime(end_ts))
index_names.reverse()
xnr_puid = load_wxxnr_redis_data(wxbot_id=wxbot_id, items=['puid'])['puid']
query_body = {
"query": {
"filtered":{
"filter":{
"bool":{
"must":[
{"term":{"xnr_id": xnr_puid}},
{"term":{"sensitive_flag": 1}},
{'term':{'msg_type':'Text'.lower()}}
]
}
}
}
},
"aggs":{
"sen_users":{
"terms":{"field": "speaker_id"}
}
},
"sort":{"timestamp":{"order":"desc"}}
}
sensitive_users = {}
for index_name in index_names:
try:
search_result = es_xnr.search(index=index_name, doc_type=wx_group_message_index_type,body=query_body)
if search_result:
res = search_result['aggregations']['sen_users']['buckets']
docs = search_result['hits']['hits']
for r in res:
groups_list = []
speaker_id = r['key']
count = r['doc_count']
for doc in docs:
if doc['_source']['speaker_id'] == speaker_id:
groups_list.append(doc['_source']['group_name'])
if speaker_id in sensitive_users:
#update: groups&count。因为是倒序查询，所以last_speak_ts在最初创建的时候就是最终的值，不需要更新。
sensitive_users[speaker_id]['count'] += count
sensitive_users[speaker_id]['groups_list'].extend(groups_list)
else:
#匹配第一条即可
for doc in docs:
if doc['_source']['speaker_id'] == speaker_id:
nickname = doc['_source']['speaker_name']
last_speak_ts = doc['_source']['timestamp']
break
sensitive_users[speaker_id] = {
'nickname': nickname,
'count': count,
'last_speak_ts': last_speak_ts,
'groups_list': groups_list
}
except Exception,e:
pass
def get_modify_userinfo(task_detail):
item_dict = {}
nick_name = task_detail['nick_name']
location_list = task_detail['location'].encode('utf-8').split('，')
try:
item_dict['location_province'] = location_list[0]
item_dict['location_city'] = location_list[1]
except:
item_dict['location_province'] = location_list[0]
item_dict['location_city'] = location_list[0]
item_dict['description'] = task_detail['description']
gender = task_detail['gender']
if gender == u'男':
item_dict['gender'] = 'man'
else:
item_dict['gender'] = 'woman'
age = task_detail['age']
birth_year = time.localtime().tm_year - int(age)
month = '%02d' % random.randint(0, 13)
day = '%02d' % random.randint(0, 29)
item_dict['birth'] = [str(birth_year), month, day]
query_body = {
'query': {
'filtered': {
'filter': {
'term': {
'nick_name': nick_name
}
}
}
}
}
es_results = es.search(index=fb_xnr_index_name,
doc_type=fb_xnr_index_type,
body=query_body)['hits']['hits']
xnr_result = es_results[0]['_source']
try:
fb_mail_account = xnr_result['fb_mail_account']
except:
fb_mail_account = ''
try:
fb_phone_account = xnr_result['fb_phone_account']
except:
fb_phone_account = ''
if fb_mail_account:
account_name = fb_mail_account
else:
account_name = fb_phone_account
password = xnr_result['password']
uid = xnr_result['uid']
try:
result = change_userinfo(account_name, password, uid, item_dict)
except:
result = False
return result
def get_show_example_model():
es_results = es.search(index=fb_example_model_index_name,doc_type=fb_example_model_index_type,\
body={'query':{'match_all':{}}})['hits']['hits']
result_all = []
for result in es_results:
result = result['_source']
result_all.append(result)
return result_all
def utils_show_report_content(wxbot_id, report_type, period, startdate,
enddate):
start_ts, end_ts, period = dump_date(period, startdate, enddate)
xnr_puid = load_wxxnr_redis_data(wxbot_id=wxbot_id, items=['puid'])['puid']
result = []
query_body = {
'query': {
'bool': {
'must': [{
'term': {
'report_type': report_type
}
}, {
'range': {
'report_time': {
'gte': start_ts,
'lte': end_ts
}
}
}, {
'term': {
'xnr_user_no': wxbot_id
}
}, {
'term': {
'xnr_puid': xnr_puid
}
}]
}
},
'size': MAX_SEARCH_SIZE,
'sort': [{
'report_time': {
'order': 'desc'
}
}]
}
try:
wx_report_management_mappings()
es_result = es_xnr.search(index=wx_report_management_index_name,
doc_type=wx_report_management_index_type,
body=query_body)['hits']['hits']
if es_result:
for item in es_result:
try:
data = item['_source']
data['_id'] = item['_id']
report_content = eval(item['_source']['report_content'])
data['sensitive_value'] = report_content['sensitive_value']
data['text'] = report_content['text']
data['sensitive_words_string'] = report_content[
'sensitive_words_string'].decode('utf8')
data.pop('report_content')
result.append(data)
except:
pass
except Exception, e:
print 'wx_report_management Exception: ', str(e)
def get_community_warning(xnr_user_no, community_id, start_time, end_time):
if S_TYPE == 'test':
time_gap = end_time - start_time
end_time = datetime2ts(WEIBO_COMMUNITY_DATE)
start_time = end_time - time_gap
else:
pass
weibo_trace_community_index_name = weibo_trace_community_index_name_pre + xnr_user_no.lower(
)
# print 'weibo_trace_community_index_name:',weibo_trace_community_index_name
print 'time:', ts2datetime(start_time), ts2datetime(end_time)
query_body = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'term': {
'community_id': community_id
}
}, {
'range': {
'trace_time': {
'gte': start_time,
'lte': end_time
}
}
}]
}
}
}
},
'size': 30,
'sort': {
'trace_time': {
'order': 'asc'
}
}
}
try:
community_result = es_xnr.search(index=weibo_trace_community_index_name,\
doc_type=weibo_trace_community_index_type,body=query_body)['hits']['hits']
result = []
# print 'community_result::',community_result
for item in community_result:
result.append(item['_source'])
except:
result = []
if result:
#对预警内容进行融合
result.sort(key=lambda k: (k.get('trace_time', 0)), reverse=False)
warning_result = get_warning_orgnize(result)
else:
warning_result = get_newcommunity_warning(xnr_user_no, community_id,
start_time, end_time)
return warning_result
def show_authority_list():
query_body = {'query': {'match_all': {}}, 'size': MAX_VALUE}
result = es.search(index=weibo_authority_management_index_name,
doc_type=weibo_authority_management_index_type,
body=query_body)['hits']['hits']
results = []
for item in result:
results.append(item['_source'])
return results
def get_show_domain():
domain_name_dict = {}
query_body = {'query':{'match_all':{}},'size':MAX_SEARCH_SIZE}
es_results = es.search(index=tw_domain_index_name,doc_type=tw_domain_index_type,body=query_body)['hits']['hits']
if es_results:
for result in es_results:
result = result['_source']
domain_name_dict[result['domain_pinyin']] = result['domain_name']
return domain_name_dict
def get_show_domain():
domain_name_dict = {}
query_body = {'query':{'term':{'compute_status':3}},'size':MAX_SEARCH_SIZE}
es_results = es.search(index=weibo_domain_index_name,doc_type=weibo_domain_index_type,body=query_body)['hits']['hits']
if es_results:
for result in es_results:
result = result['_source']
domain_name_dict[result['domain_pinyin']] = result['domain_name']
return domain_name_dict
def utils_get_safe(wxbot_id, period, startdate, enddate):
start_ts, end_ts, period = dump_date(period, startdate, enddate)
current_timestamp = int(time.time())
current_date = ts2datetime(current_timestamp)
if period == 0: #获取今天的数据
current_time = datetime2ts(current_date)
last_date = ts2datetime(current_time-DAY)
speak_dict = {}
speak_dict['speak_day'] = {}
speak_dict['speak_total'] = {}
xnr_puid = load_wxxnr_redis_data(wxbot_id=wxbot_id, items=['puid'])['puid']
#获取xnr今日发言总数
today_count = 0
query_body = {
'query':{
'bool':{
'must':[
{'term':{'speaker_id': xnr_puid}},
{'term':{'xnr_id':xnr_puid}}
]
}
}
}
today_index_name = wx_group_message_index_name_pre + current_date
try:
today_count_result = es_xnr.count(index=today_index_name,doc_type=wx_group_message_index_type,body=query_body)
if today_count_result['_shards']['successful'] != 0:
today_count = today_count_result['count']
except Exception,e:
pass
#print 'today_count Exception: ', str(e)
#获取xnr历史发言总数
total_count = 0
total_query_body = {
'query':{
'bool':{
'must':[
{'term':{'xnr_user_no': wxbot_id}},
{'term':{'puid':xnr_puid}},
{'term':{'date_time':last_date}}
]
}
}
}
total_index_name = wx_xnr_history_count_index_name
try:
total_count_result = es_xnr.search(index=total_index_name,doc_type=wx_xnr_history_count_index_type,body=total_query_body)
if total_count_result['_shards']['successful'] != 0:
total_count = total_count_result['hits']['hits'][0]['_source']['total_post_num']
except Exception,e:
pass
def lookup_hot_posts(start_time,end_time,weiboxnr_id,classify_id,order_id):
#weiboxnr_id,classify_id暂时不用
print "weiboxnr_id------------------------------------------------------------"
print weiboxnr_id
print 'post request start!!!!!'
weibo_sensitive_post_index_name_list = []
days_num = int((end_time - start_time)/DAY) + 1
print 'days_num = int((end_time - start_time)/DAY) + 1-=-=-='
print end_time, start_time, DAY
print days_num
for i in range(0,days_num):
date_ts = end_time - i*DAY
print 'date_ts, end_time, i, DAY-=-=-=-=-=-=-=-=-=-=-'
print date_ts, end_time, i, DAY
index_name = weibo_sensitive_post_index_name_pre + ts2datetime(date_ts)
print index_name
if es_xnr.indices.exists(index=index_name):
weibo_sensitive_post_index_name_list.append(index_name)
print 'has append -----------0000000000000000000000000000'
print 'has not append -----------0000000000000000000000000000'
sort_condition_list = []
if order_id==1: #按时间排序
sort_condition_list=[{'timestamp':{'order':'desc'}}]
elif order_id==2: #按热度排序
sort_condition_list=[{'retweeted':{'order':'desc'}}]
elif order_id==3: #按敏感度排序
sort_condition_list=[{'sensitive':{'order':'desc'}}]
query_body={
'query':{
'filtered':{
'filter':{
'bool':{
'must':[{'range':{'timestamp':{'gte':int(start_time),'lt':int(end_time)}}}]
}
}
}
},
'size':HOT_WEIBO_NUM,
'sort':sort_condition_list
}
try:
es_result=es_xnr.search(index=weibo_sensitive_post_index_name_list,doc_type=weibo_sensitive_post_index_type,\
body=query_body)['hits']['hits']
hot_result=[]
for item in es_result:
hot_result.append(item['_source'])
except:
hot_result=[]
return hot_result
def utils_search_by_group_puid(wxbot_id,
group_puid,
period,
startdate='',
enddate=''):
start_ts, end_ts, period = dump_date(period, startdate, enddate)
index_names = get_wx_groupmessage_index_list(ts2datetime(start_ts),
ts2datetime(end_ts))
index_names.reverse()
xnr_puid = load_wxxnr_redis_data(wxbot_id=wxbot_id, items=['puid'])['puid']
query_body = {
"query": {
"filtered": {
"filter": {
"bool": {
"must": [
{
"term": {
"xnr_id": xnr_puid
}
},
{
'term': {
'group_id': group_puid
}
},
{
"terms": {
"msg_type": [
'Text'.lower(), 'Picture'.lower(),
'Recording'.lower()
]
}
},
]
}
}
}
},
"size": MAX_VALUE,
"sort": {
"timestamp": {
"order": "desc"
}
}
}
results = []
for index_name in index_names:
try:
search_result = es_xnr.search(index=index_name,
doc_type=wx_group_message_index_type,
body=query_body)
if search_result:
results.extend(search_result['hits']['hits'])
except Exception, e:
pass
def get_tw_xnr_no():
user_no_max = 0
if not r.exists(tw_xnr_max_no): #如果当前redis没有记录，则去es数据库查找补上
es_results = es.search(index=tw_xnr_index_name,doc_type=tw_xnr_index_type,body={'query':{'match_all':{}},\
'sort':{'user_no':{'order':'desc'}}})['hits']['hits']
if es_results:
user_no_max = es_results[0]['_source']['user_no']
else: #如果当前redis有记录，则取用并更新
user_no_max = int(r.get(tw_xnr_max_no))
return user_no_max
def search_by_xnr_number(xnr_qq_number, current_date,group_qq_name):
group_qq_name_list = group_qq_name.encode('utf-8').split('，')
# 用于显示操作页面初始的所有群历史信息
query_body = {
"query": {
"filtered":{
"filter":{
"bool":{
"must":[
{"term":{"xnr_qq_number":xnr_qq_number}},
{'terms':{'qq_group_nickname':group_qq_name_list}}
]
}
}
}
},
"size": MAX_VALUE,
"sort":{"timestamp":{"order":"desc"}}
}
enddate = current_date
startdate = ts2datetime(datetime2ts(enddate)-group_message_windowsize*DAY)
index_names = get_groupmessage_index_list(startdate,enddate)
print 'index_names::',index_names
index_names.reverse()
results = {}
for index_name in index_names:
# if not es_xnr.indices.exsits(index=index_name):
# continue
try:
result = es_xnr.search(index=index_name, doc_type=group_message_index_type,body=query_body)
if results != {}:
results['hits']['hits'].extend(result['hits']['hits'])
else:
results=result #.copy()
except:
pass
# results_new = []
# for index_name in index_names:
# try:
# es_results = es_xnr.search(index=index_name, doc_type=group_message_index_type,body=query_body)['hits']['hits']
# print 'es_results::',es_results
# for es_result in es_results:
# es_result = es_result['_source']
# results_new.append(es_result)
# except:
# continue
return results
def show_reportcontent_new(report_type, start_time, end_time):
query_condition = []
# report_type_content=json.loads(report_type)
# print 'report_type_content:',report_type_content
if report_type:
query_condition.append({'terms': {'report_type': report_type}})
else:
pass
query_condition.append(
{'range': {
'report_time': {
'gte': start_time,
'lte': end_time
}
}})
query_body = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': query_condition
}
}
}
},
'size': MAX_SEARCH_SIZE,
'sort': {
'report_time': {
'order': 'desc'
}
}
}
report_management_index_name = get_xnr_reportment_index_listname(
weibo_report_management_index_name_pre, start_time, end_time)
result = []
#print 'time::',start_time,end_time
#print 'index::',report_management_index_name
try:
if report_management_index_name:
results = es_xnr.search(
index=report_management_index_name,
doc_type=weibo_report_management_index_type,
body=query_body)['hits']['hits']
else:
results = []
for item in results:
item['_source']['_id'] = item['_id']
item['_source']['report_content'] = json.loads(
item['_source']['report_content'])
result.append(item['_source'])
except:
result = []
return result
def get_speaker_info(qq_nick, index_name):
print 'qq_nick:', qq_nick
query_body = {
"query": {
"filtered": {
"filter": {
"bool": {
"must": [{
"term": {
"speaker_nickname": qq_nick
}
}, {
"term": {
"sensitive_flag": 1
}
}]
}
}
}
},
"size": MAX_VALUE,
"sort": {
"timestamp": {
"order": "desc"
}
}
}
result = es_xnr.search(index=index_name,
doc_type=group_message_index_type,
body=query_body)['hits']['hits']
results = {}
# print 'result:',result
source = result[0]['_source']
if source != []:
results['qq_nick'] = source['speaker_nickname']
results['qq_number'] = source['speaker_qq_number']
results['last_speak_ts'] = source['timestamp']
results['qq_groups'] = {
source['qq_group_number']: source['qq_group_nickname']
}
for item in result:
source = item['_source']
text_item = [
source['text'], source['timestamp'],
source['sensitive_words_string']
]
#print 'text_item:', text_item
try:
results['text'].append(text_item)
except:
results['text'] = [text_item]
#print 'final results:', results
return results
def search_by_xnr_number(xnr_qq_number, current_date):
# 用于显示操作页面初始的所有群历史信息
query_body = {
"query": {
"filtered": {
"filter": {
"bool": {
"must": [{
"term": {
"xnr_qq_number": xnr_qq_number
},
"term": {
"sensitive_flag": 1
}
}]
}
}
}
},
"size": MAX_VALUE,
"sort": {
"sensitive_value": {
"order": "desc"
}
}
}
enddate = current_date
startdate = ts2datetime(
datetime2ts(enddate) - group_message_windowsize * DAY)
index_names = get_groupmessage_index_list(startdate, enddate)
# print index_names
results = []
for index_name in index_names:
# if not es_xnr.indices.exsits(index=index_name):
# continue
try:
result = es_xnr.search(index=index_name,
doc_type=group_message_index_type,
body=query_body)['hits']['hits']
# if results != {}:
# results['hits']['hits'].extend(result['hits']['hits'])
# else:
# results=result.copy()
if result:
for item in result:
item['_source']['_id'] = item['_id']
results.append(item['_source'])
else:
pass
except:
pass
# print 'results:',results
return results
def search_by_period(xnr_qq_number, startdate, enddate, group_qq_name):
group_qq_name_list = group_qq_name.encode('utf-8').split('，')
results = {}
query_body = {
"query": {
"filtered": {
"filter": {
"bool": {
"must": [{
"term": {
"xnr_qq_number": xnr_qq_number
}
}, {
'terms': {
'qq_group_nickname': group_qq_name_list
}
}]
}
}
}
},
"size": MAX_VALUE,
"sort": {
"timestamp": {
"order": "desc"
}
}
}
# es.search(index=”flow_text_2013-09-02”, doc_type=”text”, body=query_body)
index_names = get_groupmessage_index_list(startdate, enddate)
index_names.reverse()
#print 'index_names::',index_names
for index_name in index_names:
# if not es_xnr.indices.exsits(index_name):
# continue
print index_name
try:
result = es_xnr.search(index=index_name,
doc_type=group_message_index_type,
body=query_body)
print result
if results != {}:
results['hits']['hits'].extend(result['hits']['hits'])
else:
results = result.copy()
except:
pass
if results == {}:
results = {'hits': {'hits': []}}
return results
def show_date_warming(today_time):
query_body = {
'query': {
'match_all': {}
},
'size': MAX_VALUE,
'sort': {
'date_time': {
'order': 'asc'
}
}
}
result = es_xnr.search(index=weibo_date_remind_index_name,
doc_type=weibo_date_remind_index_type,
body=query_body)['hits']['hits']
#取出预警时间进行处理
if S_TYPE == 'test':
today_time = 1480176000
else:
pass
# print '^^^^', len(result)
#print today_time
date_warming_result = []
for item in result:
#计算距离日期
date_time = item['_source']['date_time']
year = ts2yeartime(today_time)
warming_date = year + '-' + date_time
today_date = ts2datetime(today_time)
countdown_num = (datetime2ts(warming_date) -
datetime2ts(today_date)) / DAY
'''
if countdown_num<0:
new_warming_year=str(int(year)+1)
new_warming_date=new_warming_year+'-'+date_time
countdown_numday=(datetime2ts(new_warming_date)-datetime2ts(today_date))/DAY
item['_source']['countdown_days']=countdown_numday
else:
'''
item['_source']['countdown_days'] = countdown_num
if abs(countdown_num) < WARMING_DAY:
#根据给定的关键词查询预警微博
keywords = item['_source']['keywords']
item['_source'][
'weibo_date_warming_content'] = lookup_weibo_date_warming(
keywords, today_time)
date_warming_result.append(item['_source'])
else:
pass
return date_warming_result
def cron_compute_mark_wx():
current_time = int(time.time() - DAY)
current_date = ts2datetime(current_time)
current_time_new = datetime2ts(current_date)
#加载数据库中虚拟人信息
xnr_query_body = {'query': {'match_all': {}}, 'size': MAX_VALUE}
xnr_results = es_xnr.search(index=wx_xnr_index_name,
doc_type=wx_xnr_index_type,
body=xnr_query_body)['hits']['hits']
flag = False
for result in xnr_results:
xnr_user_no = result['_source']['xnr_user_no']
puid = result['_source']['puid']
#计算
influence_dict = get_influence_at_num(puid)
penetration_dict = get_penetration_num(xnr_user_no)
safe_dict = wx_history_count(xnr_user_no, puid, current_time_new)
#整理
_id = xnr_user_no + '_' + current_date
xnr_user_detail = {
'influence': influence_dict['mark'],
'penetration': penetration_dict['mark'],
'safe': safe_dict['mark'],
'daily_be_at_num': influence_dict['daily_be_at_num'],
'total_be_at_num': influence_dict['total_be_at_num'],
'daily_sensitive_num': penetration_dict['sensitive_info'],
'total_post_num': safe_dict['total_post_num'],
'daily_post_num': safe_dict['daily_post_num'],
'date_time': current_date,
'timestamp': current_time_new,
'xnr_user_no': xnr_user_no,
'puid': puid
}
#并存储
wx_xnr_history_count_mappings() #先确保数据库存在
try:
if es_xnr.exists(index=wx_xnr_history_count_index_name,
doc_type=wx_xnr_history_count_index_type,
id=_id):
msg = es_xnr.update(index=wx_xnr_history_count_index_name,
doc_type=wx_xnr_history_count_index_type,
body={'doc': xnr_user_detail},
id=_id)
else:
msg = es_xnr.index(index=wx_xnr_history_count_index_name,
doc_type=wx_xnr_history_count_index_type,
id=_id,
body=xnr_user_detail)
flag = True
print msg
except Exception, e:
print 'cron_compute_mark_wx Exception: ', str(e)
return False
def get_show_example_model():
#print '!!!!!',{'query':{'term':{'xnr_user_no':xnr_user_no}}}
es_results = es.search(index=weibo_example_model_index_name,doc_type=weibo_example_model_index_type,\
body={'query':{'match_all':{}}})['hits']['hits']
result_all = []
for result in es_results:
result = result['_source']
result_all.append(result)
return result_all
def get_xnr_info(task_detail):
nick_name = task_detail['nick_name']
query_body = {
'query':{
'filtered':{
'filter':{
'term':{'nick_name':nick_name}
}
}
}
}
es_results = es.search(index=tw_xnr_index_name,doc_type=tw_xnr_index_type,body=query_body)['hits']['hits']
return es_results
def get_nick_name_unique(nick_name):
query_body = {
'query':{
'term':{'nick_name':nick_name}
}
}
es_profile_results = es_user_profile.search(index=profile_index_name,doc_type=profile_index_type,body=query_body)['hits']['hits']
es_xnr_results = es.search(index=tw_xnr_index_name,doc_type=tw_xnr_index_type,body=query_body)['hits']['hits']
if es_profile_results and es_xnr_results:
mark = False
else:
mark = True
return mark
def search_by_period(xnr_qq_number, startdate, enddate):
results = []
query_body = {
"query": {
"filtered": {
"filter": {
"bool": {
"must": [{
"term": {
"xnr_qq_number": xnr_qq_number
},
"term": {
"sensitive_flag": 1
}
}]
}
}
}
},
"size": MAX_VALUE,
"sort": {
"timestamp": {
"order": "desc"
}
}
}
# es.search(index=”flow_text_2013-09-02”, doc_type=”text”, body=query_body)
index_names = get_groupmessage_index_list(startdate, enddate)
for index_name in index_names:
# if not es_xnr.indices.exsits(index_name):
# continue
try:
result = es_xnr.search(index=index_name,
doc_type=group_message_index_type,
body=query_body)['hits']['hits']
# if results != {}:
# results['hits']['hits'].extend(result['hits']['hits'])
# else:
# results=result.copy()
if result:
for item in result:
item['_source']['_id'] = item['_id']
results.append(item['_source'])
else:
pass
except:
pass
# if results == {}:
# results={'hits':{'hits':[]}}
return results
def get_penetration_num(xnr_user_no):
current_timestamp = int(time.time() - DAY)
current_date = ts2datetime(current_timestamp)
current_time = datetime2ts(current_date)
xnr_data = load_wxxnr_redis_data(wxbot_id=xnr_user_no,
items=['puid', 'groups_list'])
puid = xnr_data['puid']
group_list = xnr_data['groups_list']
#查询1
sensitive_value = 0
wx_group_message_index_name = wx_group_message_index_name_pre + current_date
query_body_info = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'terms': {
'group_id': group_list
}
}, {
'range': {
'sensitive_value': {
'gte': -1
}
}
}]
}
}
}
},
'aggs': {
'avg_sensitive': {
'avg': {
'field': 'sensitive_value'
}
}
}
}
try:
es_sensitive_result = es_xnr.search(
index=wx_group_message_index_name,
doc_type=wx_group_message_index_type,
body=query_body_info)['aggregations']
sensitive_value = es_sensitive_result['avg_sensitive']['value']
if sensitive_value == None:
sensitive_value = 0
except Exception, e:
print 'sensitive_value Exception: ', str(e)
def show_hidden_expression():
query_body={
'query':{
'match_all':{}
},
'size':MAX_VALUE,
'sort':{'create_time':{'order':'desc'}}
}
result=es.search(index=weibo_hidden_expression_index_name,doc_type=weibo_hidden_expression_index_type,body=query_body)['hits']['hits']
results=[]
for item in result:
item['_source']['id']=item['_id']
results.append(item['_source'])
return results
def show_sensitive_words_default():
query_body={
'query':{
'match_all':{}
},
'size':MAX_SEARCH_SIZE,
'sort':{'create_time':{'order':'desc'}}
}
result=es.search(index=weibo_sensitive_words_index_name,doc_type=weibo_sensitive_words_index_type,body=query_body)['hits']['hits']
results=[]
for item in result:
item['_source']['id']=item['_id']
results.append(item['_source'])
return results
def get_user_text(qq_nick, qq_groups, last_speak_ts):
print 'qq_nick:', qq_nick
query_body = {
"query": {
"filtered": {
"filter": {
"bool": {
"must": [{
"term": {
"speaker_nickname": qq_nick
}
}, {
"term": {
"sensitive_flag": 1
}
}, {
"term": {
"qq_group_nickname": qq_groups
}
}]
}
}
}
},
"size": MAX_VALUE,
"sort": {
"timestamp": {
"order": "desc"
}
}
}
index_name = group_message_index_name_pre + ts2datetime(last_speak_ts)
result = es_xnr.search(index=index_name,
doc_type=group_message_index_type,
body=query_body)['hits']['hits']
results = {}
for item in result:
source = item['_source']
text_item = [
source['text'], source['timestamp'],
source['sensitive_words_string']
]
#print 'text_item:', text_item
try:
results['text'].append(text_item)
except:
results['text'] = [text_item]
#print 'final results:', results
return results
def get_all_ports():
query_body = {"query": {"match_all": {}}, "size": MAX_VALUE}
try:
result = es_xnr.search(index=wx_xnr_index_name,
doc_type=wx_xnr_index_type,
body=query_body)['hits']['hits']
except:
return []
results = []
if result != []:
for item in result:
if item['_source']['wxbot_port'] not in results:
results.append(int(item['_source']['wxbot_port']))
return results
def search_qq_xnr(qq_number):
query_body = {
"query": {
"filtered":{
"filter":{
"term":{"qq_number": qq_number}
}
}
},
'size':MAX_VALUE
}
result = es_xnr.search(index=qq_xnr_index_name, doc_type=qq_xnr_index_type, body=query_body)
return result
def show_report_content():
query_body={
'query':{
'match_all':{}
},
'size':MAX_SEARCH_SIZE,
'sort':{'report_time':{'order':'desc'}}
}
results=es_xnr.search(index=weibo_report_management_index_name,doc_type=weibo_report_management_index_type,body=query_body)['hits']['hits']
result=[]
for item in results:
item['_source']['_id']=item['_id']
item['_source']['report_content']=json.loads(item['_source']['report_content'])
result.append(item['_source'])
return result
