def set_groups(self, group_list):
#注册监听群消息的函数, group_list是需要监听消息的群组的puid列表
if group_list: #['']也是True
#判断下传过来的groups_list是否有误
try:
groups_nickname = []
if group_list == ['']:
group_list = []
else:
for group_puid in group_list:
gs = self.groups(update=True).search(puid=group_puid)
if not gs:
return 0
groups_nickname.append(gs[0].name)
self.groups_list = group_list
#print 'MyBot: set_groups: self.groups_list: ', self.groups_list
groups_data = {
'wx_groups_num': len(groups_nickname),
'wx_groups_nickname': groups_nickname,
'wx_groups_puid': self.groups_list
}
self.change_wxxnr_redis_data(
{'groups_list': ','.join(self.groups_list)})
es_xnr.update(index=wx_xnr_index_name,
doc_type=wx_xnr_index_type,
id=self.wxbot_id,
body={'doc': groups_data})
except Exception, e:
print e
return 0
def lookup_id_delete(xnr_user_no,start_time,end_time,mark=False):
query_body={
'query':{
'filtered':{
'filter':{
'bool':{
'must':[
{'term':{'community_name':'心灵_赞赞'}},
{'range':{
'trace_time':{
'gte':start_time,
'lte':end_time
}
}}
]
}
}
}
},
'size':100
}
id_list = []
# try:
temp_results=es_xnr.search(index=weibo_trace_community_index_name_pre + xnr_user_no,doc_type=weibo_trace_community_index_type,body=query_body)['hits']['hits']
for item in temp_results:
print item['_id'],item['_source']['community_id']
#commu_id = 'WXNR0009_2018-04-29幸福_有希'
max_sensitive = item['_source']['max_sensitive']
if mark and (max_sensitive > 100):
es_xnr.update(index=weibo_trace_community_index_name_pre + xnr_user_no,doc_type=weibo_trace_community_index_type,id=item['_id'],body={'doc':{'max_sensitive':100.0000}})
print 'delete!!!!______'+item['_id']
def save_subopnion_results(xnr_user_no, mid, task_id, sub_opinion_results):
#try:
item_exist = dict()
item_exist['xnr_user_no'] = xnr_user_no
item_exist['mid'] = mid
item_exist['subopinion_tw'] = json.dumps(sub_opinion_results)
#print 'sub_opinion_results::',sub_opinion_results
es.index(index=tw_hot_subopinion_results_index_name,doc_type=tw_hot_subopinion_results_index_type,\
id=task_id,body=item_exist)
item_task = dict()
item_task['compute_status'] = 2 ## 保存子观点结果
print 'tw_hot_keyword_task_index_name...', tw_hot_keyword_task_index_name
es.update(index=tw_hot_keyword_task_index_name,doc_type=tw_hot_keyword_task_index_type,\
id=task_id,body={'doc':item_task})
mark = True
#except Exception,e:
# mark = False
return mark
def tw_save_to_fans_follow_ES(xnr_user_no,uid,follow_type,trace_type):
try:
results = es_xnr.get(index=fb_xnr_fans_followers_index_name,doc_type=fb_xnr_fans_followers_index_type,\
id=xnr_user_no)
results = results["_source"]
except:
results = []
if follow_type == 'follow':
if trace_type == 'trace_follow':
try:
if results:
# 添加追随关注
try:
trace_follow_uids = results['trace_follow_list']
trace_follow_uids_set = set(trace_follow_uids)
trace_follow_uids_set.add(uid)
trace_follow_uids = list(trace_follow_uids_set)
except:
trace_follow_uids = [uid]
# # 添加普通关注
# try:
# followers_uids = results['followers_list']
# followers_uids_set = set(followers_uids)
# followers_uids_set.add(uid)
# followers_uids = list(followers_uids_set)
# except:
# followers_uids = [uid]
# results['followers_list'] = followers_uids
results['trace_follow_list'] = trace_follow_uids
es_xnr.update(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\
id=xnr_user_no,body={'doc':results})
else:
results['trace_follow_list'] = [uid]
results['followers_list'] = [uid]
es_xnr.index(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\
body=results,id=xnr_user_no)
except:
return False
elif follow_type == 'unfollow':
try:
followers_uids = results['trace_follow_pre_list']
followers_uids = list(set(followers_uids).difference(set([uid])))
results['trace_follow_pre_list'] = followers_uids
es_xnr.update(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\
id=xnr_user_no,body={'doc':results})
except:
return False
return True
def save_detect_results(detect_results, decect_task_information):
mark = False
task_id = decect_task_information['domain_pinyin']
try:
item_exist = es_xnr.get(index=fb_domain_index_name,
doc_type=fb_domain_index_type,
id=task_id)['_source']
item_exist['group_size'] = len(detect_results)
item_exist['member_uids'] = detect_results
item_exist['compute_status'] = 1 # 存入uid
es_xnr.update(index=fb_domain_index_name,
doc_type=fb_domain_index_type,
id=task_id,
body={'doc': item_exist})
except Exception, e:
print e
item_exist = dict()
item_exist['xnr_user_no'] = decect_task_information['xnr_user_no']
item_exist['domain_pinyin'] = decect_task_information['domain_pinyin']
item_exist['domain_name'] = decect_task_information['domain_name']
item_exist['create_type'] = decect_task_information['create_type']
item_exist['create_time'] = decect_task_information['create_time']
item_exist['submitter'] = decect_task_information['submitter']
item_exist['description'] = decect_task_information['description']
item_exist['remark'] = decect_task_information['remark']
item_exist['group_size'] = len(detect_results)
item_exist['member_uids'] = detect_results
item_exist['compute_status'] = 1 # 存入uid
es_xnr.index(index=fb_domain_index_name,
doc_type=fb_domain_index_type,
id=task_id,
body=item_exist)
def save_role_feature_analysis(role_results,role_label,domain,role_id):
mark = False
try:
item_exist = es_xnr.get(index=weibo_role_index_name,doc_type=weibo_role_index_type,id=role_id)['_source']
item_exist['role_pinyin'] = role_id
item_exist['role_name'] = role_label
item_exist['domains'] = domain
item_exist['personality'] = role_results['personality']
item_exist['political_tendency'] = role_results['political_tendency']
item_exist['geo'] = role_results['geo']
item_exist['active_time'] = role_results['active_time']
item_exist['day_post_num'] = role_results['day_post_num']
es_xnr.update(index=weibo_role_index_name,doc_type=weibo_role_index_type,id=role_id,body={'doc':item_exist})
except Exception, e:
item_exist = dict()
item_exist['role_pinyin'] = role_id
item_exist['role_name'] = role_label
item_exist['domains'] = domain
item_exist['personality'] = role_results['personality']
item_exist['political_tendency'] = role_results['political_tendency']
item_exist['geo'] = role_results['geo']
item_exist['active_time'] = role_results['active_time']
item_exist['day_post_num'] = role_results['day_post_num']
es_xnr.index(index=weibo_role_index_name,doc_type=weibo_role_index_type,id=role_id,body=item_exist)
def save_group_description_results(group_results,decect_task_information):
mark = False
task_id = decect_task_information['domain_pinyin']
try:
item_exist = es_xnr.get(index=weibo_domain_index_name,doc_type=weibo_domain_index_type,id=task_id)['_source']
item_exist['role_distribute'] = json.dumps(group_results['role_distribute'])
item_exist['top_keywords'] = json.dumps(group_results['top_keywords'])
item_exist['political_side'] = json.dumps(group_results['political_side'])
item_exist['topic_preference'] = json.dumps(group_results['topic_preference'])
item_exist['compute_status'] = 2 # 存入群体描述
es_xnr.update(index=weibo_domain_index_name,doc_type=weibo_domain_index_type,id=task_id,body={'doc':item_exist})
except Exception, e:
item_exist = dict()
item_exist['domain_pinyin'] = json.dumps(decect_task_information['domain_pinyin'])
item_exist['domain_name'] = json.dumps(decect_task_information['domain_name'])
item_exist['create_type'] = json.dumps(decect_task_information['create_type'])
item_exist['create_time'] = json.dumps(decect_task_information['create_time'])
item_exist['submitter'] = json.dumps(decect_task_information['submitter'])
item_exist['remark'] = json.dumps(decect_task_information['remark'])
item_exist['role_distribute'] = json.dumps(group_results['role_distribute'])
item_exist['top_keywords'] = json.dumps(group_results['top_keywords'])
item_exist['political_side'] = json.dumps(group_results['political_side'])
item_exist['topic_preference'] = json.dumps(group_results['topic_preference'])
item_exist['compute_status'] = 2 # 存入群体描述
es_xnr.index(index=weibo_domain_index_name,doc_type=weibo_domain_index_type,id=task_id,body=item_exist)
def save_subopnion_results(xnr_user_no, mid, task_id, sub_opinion_results):
#try:
item_exist = dict()
item_exist['xnr_user_no'] = xnr_user_no
item_exist['mid'] = mid
item_exist['subopinion_weibo'] = json.dumps(sub_opinion_results)
#print 'sub_opinion_results::',sub_opinion_results
es.index(index=weibo_hot_subopinion_results_index_name,doc_type=weibo_hot_subopinion_results_index_type,\
id=task_id,body=item_exist)
item_task = dict()
item_task['compute_status'] = 2 ## 保存子观点结果
es.update(index=weibo_hot_keyword_task_index_name,doc_type=weibo_hot_keyword_task_index_type,\
id=task_id,body={'doc':item_task})
item_sensing = dict()
item_sensing['compute_status'] = 2
result = es.get(index=social_sensing_index_name,
doc_type=social_sensing_index_type,
id=mid)
print 'result:::', result
es.update(index=social_sensing_index_name,doc_type=social_sensing_index_type,id=mid,\
body={'doc':item_sensing})
mark = True
#except Exception,e:
# mark = False
return mark
def save_content_recommendation_results(xnr_user_no, mid, task_id,
content_results):
#try:
item_exist = dict()
item_exist['xnr_user_no'] = xnr_user_no
item_exist['mid'] = mid
item_exist['content_recommend'] = json.dumps(content_results)
print '123123123'
es.index(index=tw_hot_content_recommend_results_index_name,\
doc_type=tw_hot_content_recommend_results_index_type,\
id=task_id,body=item_exist)
print '45456456'
print 'mid:::', mid
print 'task_id::', task_id
item_task = dict()
item_task['compute_status'] = 1 ## 保存内容推荐结果
es.update(index=tw_hot_keyword_task_index_name,doc_type=tw_hot_keyword_task_index_type,\
id=task_id,body={'doc':item_task})
es.update(index=social_sensing_index_name,doc_type=social_sensing_index_type,id=mid,\
body={'doc':{'compute_status':1}})
print '##########'
mark = True
# except Exception, e:
# mark = False
return mark
def save(index_name, doc_type, id, data):
if es.exists(inde=index_name, doc_type=doc_type, id=id):
es.update(inde=index_name,
doc_type=doc_type,
id=id,
body={'doc': data})
else:
es.index(inde=index_name, doc_type=doc_type, id=id, body=data)
def update_opnion_corpus(corpus_id):
try:
es_xnr.update(index=opinion_corpus_index_name,doc_type=opinion_corpus_index_type,\
id=corpus_id,body={'doc':{'status':1}})
mark = True
except:
mark = False
return mark
def followers_domain_update():
if S_TYPE == 'test':
current_time = datetime2ts(S_DATE)
else:
current_time = int(time.time())
flow_text_index_name_list = get_flow_text_index_list(current_time)
query_body = {'query': {'match_all': {}}, 'size': MAX_VALUE}
search_results = es_xnr.search(index=weibo_xnr_fans_followers_index_name,\
doc_type=weibo_xnr_fans_followers_index_type,body=query_body)['hits']['hits']
followers_list_all = []
for result in search_results:
result = result['_source']
followers_list = result['followers_list']
followers_list_all.extend(followers_list)
followers_list_all_set_list = list(set(followers_list_all))
uid_weibo_keywords_dict, keywords_dict_all_users = uid_list_2_uid_keywords_dict(
followers_list_all_set_list, flow_text_index_name_list)
uids_avtive_list = uid_weibo_keywords_dict.keys(
) # 防止关注列表中有无效uid，或者只有近期活跃的uid才有意义。
## 领域分类
r_domain = dict()
print 'uids_avtive_list::', uids_avtive_list
domain, r_domain = domain_classfiy(uids_avtive_list,
uid_weibo_keywords_dict)
print 'r_domain::', r_domain
for uid, domain in r_domain.iteritems():
domain_name = domain_en2ch_dict[domain]
_id = uid
try:
print '_id:::', _id
get_result = es_xnr.get(index=user_domain_index_name,doc_type=user_domain_index_type,\
id=_id)['_source']
get_result['domain_name'] = domain_name
get_result['update_time'] = int(time.time())
es_xnr.update(index=user_domain_index_name,doc_type=user_domain_index_type,\
id=_id,body={'doc':get_result})
except:
item_dict = {}
item_dict['uid'] = uid
item_dict['domain_name'] = domain_name
item_dict['update_time'] = int(time.time())
es_xnr.index(index=user_domain_index_name,doc_type=user_domain_index_type,\
id=_id,body=item_dict)
def tw_count2flow_text():
index_name = twitter_count_index_name_pre + '*'
query_body = {'query': {'match_all': {}}}
scan_results = scan(es,
index=index_name,
doc_type=twitter_count_index_type,
query=query_body,
size=1000)
count = 0
while 1:
try:
body_dict = {}
data = scan_results.next()
item = data['_source']
body_dict['comment'] = item['comment']
body_dict['favorite'] = item['favorite']
body_dict['share'] = item['share']
body_dict['update_time'] = item['update_time']
start_ts = datetime2ts('2017-10-10')
end_ts = datetime2ts('2017-10-25')
day_num = (end_ts - start_ts) / (24 * 3600) + 1
for i in range(day_num):
timestamp = start_ts + i * 24 * 3600
date = ts2datetime(timestamp)
flow_text_index_name = twitter_flow_text_index_name_pre + date
_id = item['tid']
try:
es.update(index=flow_text_index_name,doc_type=twitter_flow_text_index_type,\
id=_id,body={'doc':body_dict})
count += 1
if count % 1000 == 0:
print 'tw..', count
except:
continue
except StopIteration:
break
def save2opinion_corpus(task_id, opinion_results):
item_exist = dict()
item_exist['task_id'] = task_id
item_exist['corpus_results'] = json.dumps(opinion_results)
es_intel.index(index=opinion_corpus_results_index_name,doc_type=opinion_corpus_results_index_type,\
id=task_id,body=item_exist)
item_task = dict()
item_task['compute_status'] = 3 ## 保存观点语料结果，更新计算状态
es_xnr.update(index=writing_task_index_name,doc_type=writing_task_index_type,\
id=task_id, body={'doc':item_task})
def change_process_proportion(task_id, proportion):
mark = False
try:
task_exist_result = es_xnr.get(index=weibo_domain_index_name, doc_type=weibo_domain_index_type, id=task_id)['_source']
except:
task_exist_result = {}
return 'task is not exist'
if task_exist_result != {}:
task_exist_result['compute_status'] = proportion
es_xnr.update(index=weibo_domain_index_name, doc_type=weibo_domain_index_type, id=task_id, body={'doc':task_exist_result})
mark = True
return mark
def save_error_es(username, account_type):
if account_type == 'phone':
print '_+_+_+_+_+_+_+_+_+_+_+_+_+_+_+_+_错误的登录'
body = {"query":{"term":{"weibo_phone_account":username}}}
item_exist = es.search(index=weibo_xnr_index_name,doc_type=weibo_xnr_index_type, body=body)['hits']['hits']
xnr_id = item_exist[0]['_id']
print es.update(index=weibo_xnr_index_name,doc_type=weibo_xnr_index_type,id=xnr_id,body={'doc':{'verify_password':username}})
elif account_type == "mail":
print '_+_+_+_+_+_+_+_+_+_+_+_+_+_+_+_+_错误的登录'
body = {"query":{"term":{"weibo_mail_account":username}}}
item_exist = es.search(index=weibo_xnr_index_name,doc_type=weibo_xnr_index_type, body=body)['hits']['hits']
xnr_id = item_exist[0]['_id']
print es.update(index=weibo_xnr_index_name,doc_type=weibo_xnr_index_type,id=xnr_id,body={'doc':{'verify_password':username}})
print 'mail'
else:
print '_+_+_+_+_+_+_+_+_+_+_+_+_+_+_+_+_正确的登录'
body = {"query":{"term":{"weibo_mail_account":username}}}
body2 = {"query":{"term":{"weibo_phone_account":username}}}
item_exist = es.search(index=weibo_xnr_index_name,doc_type=weibo_xnr_index_type, body=body)['hits']['hits']
if item_exist:
xnr_id = item_exist[0]['_id']
print es.update(index=weibo_xnr_index_name,doc_type=weibo_xnr_index_type,id=xnr_id,body={'doc':{'verify_password':''}})
else:
item_exist = es.search(index=weibo_xnr_index_name,doc_type=weibo_xnr_index_type, body=body2)['hits']['hits']
xnr_id = item_exist[0]['_id']
print es.update(index=weibo_xnr_index_name,doc_type=weibo_xnr_index_type,id=xnr_id,body={'doc':{'verify_password':''}})
def get_tw_xnr_fans_followers():
query_body_tw = {
'query': {
'term': {
'create_status': 2
}
},
'size': MAX_SEARCH_SIZE
}
tw_xnrs = es_xnr.search(index=tw_xnr_index_name, doc_type=tw_xnr_index_type,\
body=query_body_tw)['hits']['hits']
for tw_xnr in tw_xnrs:
root_uid = tw_xnr['_source']['uid']
xnr_user_no = tw_xnr['_source']['xnr_user_no']
query_body = {
'query': {
'term': {
'root_uid': root_uid
}
},
'size': MAX_SEARCH_SIZE
}
tw_results = es_xnr.search(index=twitter_feedback_follow_index_name,doc_type=twitter_feedback_follow_index_type,\
body=query_body)['hits']['hits']
friends_list = []
for tw_result in tw_results:
uid = tw_result['_source']['uid']
friends_list.append(uid)
try:
get_results = es_xnr.get(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\
id=xnr_user_no)['_source']
es_xnr.update(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\
id=xnr_user_no,body={'doc':{'followers_list':friends_list}})
except:
es_xnr.index(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\
id=xnr_user_no,body={'followers_list':friends_list})
def dataUpdate(xnr_qq_number, today_num, history_num):
try:
es.update(index=qq_xnr_index_name,
doc_type=qq_xnr_index_type,
id=xnr_qq_number,
body={
"doc": {
"today_speak_num": today_num,
"all_speak_num": history_num
}
})
return 1
except:
print '发言数更新失败'
return 0
def save2models_text(task_id, model_text_dict):
item_exist = dict()
item_exist['task_id'] = task_id
item_exist['model_text_pos'] = model_text_dict['model_text_pos']
item_exist['model_text_neg'] = model_text_dict['model_text_neg']
item_exist['model_text_news'] = model_text_dict['model_text_news']
# 保存智能发帖模板文本结果
print 'item_exist...', item_exist
es_intel.index(index=intel_models_text_index_name,doc_type=intel_models_text_index_type,\
id=task_id,body=item_exist)
item_task = dict()
item_task['compute_status'] = 2 ## 保存智能发帖模板文本结果，更新计算状态
es_xnr.update(index=writing_task_index_name,doc_type=writing_task_index_type,\
id=task_id, body={'doc':{'compute_status':2}})
def save_tw_app_api_info(xnr_user_no, info):
try:
if es_xnr.exists(index=tw_xnr_index_name,
doc_type=tw_xnr_index_type,
id=xnr_user_no):
print es_xnr.update(index=tw_xnr_index_name,
doc_type=tw_xnr_index_type,
body={'doc': info},
id=xnr_user_no)
else:
print es_xnr.index(index=tw_xnr_index_name,
doc_type=tw_xnr_index_type,
id=xnr_user_no,
body=info)
return True
except Exception, e:
print e
return False
def save_friends(xnr_info):
friend = Friend(xnr_info['username'], xnr_info['password'])
res = friend.get_friend()
if res:
for r in res:
data = r
facebook_type = u'陌生人'
# if uid in xnr_info['friends_list']:
# facebook_type = u'好友'
data['root_uid'] = xnr_info['root_uid']
data['root_nick_name'] = xnr_info['root_nick_name']
data['facebook_type'] = facebook_type
index_name = facebook_feedback_friends_index_name
facebook_feedback_friends_mappings(index_name)
query_body = {
"query": {
"filtered": {
"filter": {
"bool": {
"must": [
{
"term": {
"uid": data['uid']
}
},
]
}
}
}
},
}
results = es.search(index=index_name,
doc_type=facebook_feedback_friends_index_type,
body=query_body)['hits']['hits']
if results:
es.update(inde=index_name,
doc_type=facebook_feedback_friends_index_type,
body={'doc': data})
else:
es.index(inde=index_name,
doc_type=facebook_feedback_friends_index_type,
body=data)
return True
def rpop_compute_intelligent_writing():
while True:
temp = r_r.rpop(writing_task_queue_name)
print 'temp:::::', temp
if not temp:
print '当前没有智能写作推荐任务'
break
task_detail = json.loads(temp)
task_id = task_detail['task_id']
#print 'task_detail::',task_detail
print '把任务从队列中pop出来......'
compute_intelligent_recommend(task_detail)
es_xnr.update(index=writing_task_index_name,doc_type=writing_task_index_type,\
id=task_id, body={'doc':{'compute_status':1}})
def rpop_expand_opinion_corpus():
while True:
temp = r_r.rpop(opinion_expand_task_queue_name)
# print 'temp:::::',temp
if not temp:
print '当前没有观点语料库扩充任务'
break
task_detail = json.loads(temp)
task_id = task_detail['corpus_pinyin']
print 'task_detail::', task_detail
print '把任务从队列中pop出来......'
spcific_opinion_corpus_expand(task_detail)
es_xnr.update(index=opinion_corpus_index_name,doc_type=opinion_corpus_index_type,\
id=task_id, body={'doc':{'status':1}})
def lookup_detail():
index_name = 'weibo_community_2018-05-20'
index_type = 'community'
task_id = 'WXNR0009_2018-04-29富贵_师兄'
result = es_xnr.update(index=index_name,doc_type=index_type,id=task_id,body={'doc':{'community_status':1}})
#ommunity_user_change = result['community_user_change']
#eturn community_user_change
#result = es_xnr.update(index=index_name,\
# doc_type=index_type,id=task_id,body={'doc':{'max_sensitive':100.0000}})
print result
def rpop_select_community():
while True:
temp = r_community.rpop(weibo_community_select_task_queue_name)
# print 'temp:::::',temp
if not temp:
print '当前没有选择社区任务'
break
task_detail = json.loads(temp)
xnr_user_no = task_detail['xnr_user_no']
datetime = task_detail['datetime']
task_id = xnr_user_no + '_' + ts2datetime(datetime)
print 'task_detail::', task_detail
print '把任务从队列中pop出来......'
mark = get_final_community(xnr_user_no, datetime)
if mark:
es_xnr.update(index=weibo_community_status_index_name,doc_type=weibo_community_status_index_type,\
id=task_id, body={'doc':{'status':1}})
def save_intelligent_opinion_results(task_id,sub_opinion_results,summary, intel_type):
try:
item_exist = dict()
item_exist['task_id'] = task_id
item_exist['subopinion_tweets'] = json.dumps(sub_opinion_results)
item_exist['summary'] = summary
# 保存子观点结果
es_intel.index(index=intel_opinion_results_index_name,doc_type=intel_type,\
id=task_id,body=item_exist)
item_task = dict()
item_task['compute_status'] = 2 ## 保存子观点结果，更新计算状态
es_xnr.update(index=writing_task_index_name,doc_type=writing_task_index_type,\
id=task_id, body={'doc':{'compute_status':2}})
mark = True
except:
mark = False
return mark
def update_warning_rank(community, trace_datetime):
weibo_community_index_name = get_community_index(trace_datetime)
# community_get = es_xnr.get(index=weibo_community_index_type,doc_type=weibo_community_index_type,id=community_id)['_source']
# warning_
try:
update_result = es_xnr.update(index=weibo_community_index_name,doc_type=weibo_community_index_type,\
id=community['community_id'],body={'doc':{'warning_rank':community['warning_rank'],\
'warning_type':community['warning_type'],'density':community['density'],'cluster':community['cluster'],\
'max_influence':community['max_influence'],'mean_influence':community['mean_influence'],\
'max_sensitive':community['max_sensitive'],'mean_sensitive':community['mean_sensitive']}})
mark = True
except:
mark = False
return mark
def save_role_feature_analysis(role_results, role_label, domain, role_id,
task_id):
mark = False
try:
item_exist = es.get(index=weibo_role_index_name,
doc_type=weibo_role_index_type,
id=role_id)['_source']
item_exist['role_pinyin'] = role_id
item_exist['role_name'] = role_label
item_exist['domains'] = domain
item_exist['personality'] = json.dumps(role_results['personality'])
item_exist['political_side'] = json.dumps(
role_results['political_side'])
item_exist['geo'] = json.dumps(role_results['geo'])
item_exist['active_time'] = json.dumps(
list(role_results['active_time']))
item_exist['day_post_num'] = json.dumps(
list(role_results['day_post_num']))
es.update(index=weibo_role_index_name,
doc_type=weibo_role_index_type,
id=role_id,
body={'doc': item_exist})
item_domain = dict()
item_domain['compute_status'] = 3 # 存入角色分析结果
es.update(index=weibo_domain_index_name,
doc_type=weibo_domain_index_type,
id=task_id,
body={'doc': item_domain})
except Exception, e:
item_exist = dict()
item_exist['role_pinyin'] = role_id
item_exist['role_name'] = role_label
item_exist['domains'] = domain
item_exist['personality'] = json.dumps(role_results['personality'])
item_exist['political_side'] = json.dumps(
role_results['political_side'])
item_exist['geo'] = json.dumps(role_results['geo'])
item_exist['active_time'] = json.dumps(
list(role_results['active_time']))
item_exist['day_post_num'] = json.dumps(
list(role_results['day_post_num']))
es.index(index=weibo_role_index_name,
doc_type=weibo_role_index_type,
id=role_id,
body=item_exist)
item_domain = dict()
item_domain['compute_status'] = 3 # 存入角色分析结果
es.update(index=weibo_domain_index_name,
doc_type=weibo_domain_index_type,
id=task_id,
body={'doc': item_domain})
def delete_xnr_followers(xnr_user_no, follower_uid):
xnr_es_result = es_xnr.get(index=weibo_xnr_fans_followers_index_name,
doc_type=weibo_xnr_fans_followers_index_type,
id=xnr_user_no)['_source']
user_no = int(xnr_user_no[-4:])
uid = xnr_es_result['uid']
fans_list = xnr_es_result['fans_list']
origin_followers_list = xnr_es_result['followers_list']
origin_followers_list.remove(follower_uid)
followers_list = origin_followers_list
try:
mark=es_xnr.update(index=weibo_xnr_fans_followers_index_name,doc_type=weibo_xnr_fans_followers_index_type,id=xnr_user_no,\
body={"doc":{'user_no':user_no,'uid':uid,'fans_list':fans_list,'followers_list':followers_list}})
mark = True
except:
mark = False
return mark
def getgroup_v2(qq_xnr):
group_dict = {}
#step0: get qqbot_port
if qq_xnr[:4] != 'QXNR':
search_result = es.search(index=qq_xnr_index_name,doc_type=qq_xnr_index_type,\
body={'query':{'term':{'qq_number':qq_xnr}}})['hits']['hits']
qq_xnr = search_result[0]['_id']
#try:
qq_xnr_es_result = es.get(index=qq_xnr_index_name,\
doc_type=qq_xnr_index_type, id=qq_xnr, _source=True)['_source']
group_info = json.loads(qq_xnr_es_result['group_info'])
qqbot_port = qq_xnr_es_result['qqbot_port']
print 'qqbot_port..', qqbot_port
p_str = 'qq ' + str(qqbot_port) + ' list group'
p = subprocess.Popen(p_str, shell=True, \
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
line_count = 0
for line in p.stdout.readlines():
line_count += 1
#print 'line.==========',line
if line_count >= 5 and line_count % 2 == 1:
item_line_list = line.split('|')
try:
#qq_group_number = str(int(item_line_list[2]))
qq_uin_number = str(int(item_line_list[7]))
#print 'qq_uin_number..',qq_uin_number
qq_group_name = item_line_list[4]
qq_mark_name = item_line_list[5]
# group_dict[qq_group_number] = qq_group_name
group_dict[qq_uin_number] = qq_group_name
# 如果uin为空，则添加进去uin，如果不为空，则更新群名（因为群名可能修改）
for key, value_dict in group_info.iteritems():
mark_name = value_dict['mark_name']
if not qq_mark_name:
if qq_mark_name == mark_name:
if not qq_group_name in value_dict['group_name']:
group_info[key]['group_name'].append(
qq_group_name)
except:
next
group_info = json.dumps(group_info)
es.update(index=qq_xnr_index_name,
doc_type=qq_xnr_index_type,
id=qq_xnr,
body={'doc': {
'group_info': group_info
}})
print 'group_dict::len..', len(group_dict)
return group_dict